.builds/arch.yml

@@ -1,25 +0,0 @@
-image: archlinux
-packages:
-  - dotnet-sdk
-  - dotnet-runtime-6.0
-  - docker
-sources:
-  - https://git.sr.ht/~cloutier/bird.makeup
-secrets:
-  -  d9970e85-5aef-4cfd-b6ed-0ccf1be5308b
-tasks:
-  - test: |
-      sudo systemctl start docker
-      sudo docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=birdsitelive -e POSTGRES_USER=birdsitelive -e POSTGRES_DB=birdsitelive postgres:15
-      cd bird.makeup/src
-      dotnet test
-  - publish-arm: |
-      cd bird.makeup/src/BirdsiteLive
-      dotnet publish --os linux --arch arm64 /t:PublishContainer -c Release
-      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest-arm
-      docker push cloutier/bird.makeup:latest-arm
-  - publish-x64: |
-      cd bird.makeup/src/BirdsiteLive
-      dotnet publish --os linux --arch x64 /t:PublishContainer -c Release
-      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest
-      docker push cloutier/bird.makeup:latest

.gitignore

@@ -91,6 +91,7 @@ StyleCopReport.xml
 *.log
 *.vspscc
 *.vssscc
+.builds
 *.pidb
 *.svclog
 *.scc
@@ -345,8 +346,9 @@ ASALocalRun/
 # BeatPulse healthcheck temp database
 healthchecksdb
 
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
 /src/BSLManager/Properties/launchSettings.json
-
-backups

Dockerfile

@@ -1,9 +1,11 @@
-FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS base
+#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
+
+FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base
 WORKDIR /app
 EXPOSE 80
 EXPOSE 443
 
-FROM mcr.microsoft.com/dotnet/sdk:6.0 AS publish
+FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish
 COPY ./src/ ./src/
 RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
 RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish
@@ -11,4 +13,4 @@ RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-conta
 FROM base AS final
 WORKDIR /app
 COPY --from=publish /app/publish .
-ENTRYPOINT ["dotnet", "BirdsiteLive.dll"]
+ENTRYPOINT ["dotnet", "BirdsiteLive.dll"]

INSTALLATION.md

@@ -1,5 +1,12 @@
 # Installation
 
+## Prerequisites 
+
+You will need a Twitter API key to make BirdsiteLIVE working. First create an **Standalone App** in the [Twitter developer portal](https://developer.twitter.com/en/portal/projects-and-apps) and retrieve the API Key and API Secret Key. 
+
+Please make sure you are using a **Standalone App** API Key and not a **Project App** API Key (that will NOT work with BirdsiteLIVE), if you don't see the **Standalone App** section, you might need to [apply for Elevated Access](https://developer.twitter.com/en/portal/products/elevated) as described in the [API documentation](https://developer.twitter.com/en/support/twitter-api/developer-account).
+
+
 ## Server prerequisites
 
 Your instance will need [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed and working. 
@@ -24,6 +31,8 @@ sudo nano docker-compose.yml
 
 * `Instance:Domain` the domain name you'll be using, for example use `birdsite.live` for the URL `https://birdsite.live`
 * `Instance:AdminEmail` the admin's email, will be displayed in the instance /.well-known/nodeinfo endpoint
+* `Twitter:ConsumerKey` the Twitter API key
+* `Twitter:ConsumerSecret` the Twitter API secret key
 
 #### Database credentials
 

README.md

@@ -1,44 +1,31 @@
-# bird.makeup
+![Test](https://github.com/NicolasConstant/BirdsiteLive/workflows/.NET%20Core/badge.svg?branch=master&event=push)
 
-[![builds.sr.ht status](https://builds.sr.ht/~cloutier/bird.makeup/commits/master/arch.yml.svg)](https://builds.sr.ht/~cloutier/bird.makeup/commits/master/arch.yml?)
+# BirdsiteLIVE
 
 ## About
 
-Bird.makeup is a way to follow Twitter users from any ActivityPub service. The aim is to make tweets appear as native a possible to the fediverse, while being as scalable as possible. The project started from BirdsiteLive, but has now been improved significantly. 
+BirdsiteLIVE is an ActivityPub bridge from Twitter, it's mostly a pet project/playground for me to handle ActivityPub concepts. Feel free to deploy your own instance (especially if you plan to follow a lot of users) since it use a proper Twitter API key and therefore will have limited calls ceiling (it won't scale, and it's by design).
 
-Compared to BirdsiteLive, bird.makeup is:
+## State of development
 
-More scalable:
+The code is pretty messy and far from a good state, since it's a playground for me the aim was to understand some AP concepts, not provide a good state-of-the-art codebase. But I might refactor it to make it cleaner. 
- - Twitter API calls are not rate-limited
- - It is possible to split the Twitter crawling to multiple servers
- - There are now integration tests for the non-official api
- - The core pipeline has been tweaked to remove bottlenecks. As of writing this, bird.makeup supports without problems more than 20k users. 
- - Twitter users with no followers on the fediverse will stop being fetched
-
-More native to the fediverse:
- - Retweets are propagated as boosts
- - Activities are now "unlisted" which means that they won't polute the public timeline, but they can still be boosted
- - WIP support for QT
-
-More modern:
- - Moved from .net core 3.1 to .net 6 which is still supported
- - Moved from postgres 9 to 15
- - Moved from Newtonsoft.Json to System.Text.Json
 
 ## Official instance 
 
-You can find the official instance here: [bird.makeup](https://bird.makeup). If you are an instance admin that prefers to not have tweets federated to you, please block the entire instance. 
+You can find an official (and temporary) instance here: [beta.birdsite.live](https://beta.birdsite.live). This instance can disapear at any time, if you want a long term instance you should install your own or use another one. 
 
-Please consider if you really need another instance before spinning up a new one, as having multiple domain makes it harder for moderators to block twitter content. 
+## Installation
+
+I'm providing a [docker build](https://hub.docker.com/r/nicolasconstant/birdsitelive). To install it on your own server, please follow [those instructions](https://github.com/NicolasConstant/BirdsiteLive/blob/master/INSTALLATION.md). More [options](https://github.com/NicolasConstant/BirdsiteLive/blob/master/VARIABLES.md) are also available.
+
+Also a [CLI](https://github.com/NicolasConstant/BirdsiteLive/blob/master/BSLManager.md) is available for adminitrative tasks.
 
 ## License
 
-Original code started from [BirdsiteLive](https://github.com/NicolasConstant/BirdsiteLive).
+This project is licensed under the AGPLv3 License - see [LICENSE](https://github.com/NicolasConstant/BirdsiteLive/blob/master/LICENSE) for details.
-
-This project is licensed under the AGPLv3 License - see [LICENSE](https://git.sr.ht/~cloutier/bird.makeup/tree/master/item/LICENSE) for details.
 
 ## Contact
 
-You can contact me via ActivityPub <a rel="me" href="https://social.librem.one/@vincent">here</a>.
+You can contact me via ActivityPub <a rel="me" href="https://fosstodon.org/@BirdsiteLIVE">here</a>.
 
 

docker-compose.yml

@@ -1,40 +1,39 @@
 version: "3"
 
-services:
+networks:
+    birdsitelivenetwork:
+        external: false
 
+services:
     server:
-        image: cloutier/bird.makeup:latest
+        image: nicolasconstant/birdsitelive:latest
         restart: always
-        container_name: birdmakeup
+        container_name: birdsitelive
         environment:
-            - Instance:Domain=bird.makeup
+            - Instance:Domain=domain.name
-            - Instance:Name=bird.makeup
             - Instance:AdminEmail=name@domain.ext
-            - Instance:ParallelTwitterRequests=50
-            - Instance:ParallelFediverseRequests=20
             - Db:Type=postgres
             - Db:Host=db
             - Db:Name=birdsitelive
             - Db:User=birdsitelive
             - Db:Password=birdsitelive
-            - Moderation:FollowersBlackListing=bae.st
+            - Twitter:ConsumerKey=twitter.api.key
+            - Twitter:ConsumerSecret=twitter.api.key
+        networks:
+            - birdsitelivenetwork
         ports:
             - "5000:80"
-        volumes:
-            - type: bind
-              source: ../key.json
-              target: /app/key.json
         depends_on:
             - db
+
     db:
-        image: postgres:15
+        image: postgres:9.6
         restart: always
         environment:
             - POSTGRES_USER=birdsitelive
             - POSTGRES_PASSWORD=birdsitelive
             - POSTGRES_DB=birdsitelive
+        networks:
+            - birdsitelivenetwork
         volumes:
-            - ../postgres15:/var/lib/postgresql/data
+            - ./postgres:/var/lib/postgresql/data
-        ports:
-            - "5432:5432"
-

sql.md

@@ -1,45 +0,0 @@
-
-# Most common servers
-
-```SQL
-SELECT COUNT(*), host FROM followers GROUP BY host ORDER BY count DESC;
-```
-
-# Most popular twitter users
-
-```SQL
-SELECT COUNT(*), acct FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY acct ORDER BY count DESC;
-```
-
-```SQL
-SELECT COUNT(*), acct, id FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id WHERE id IN ( SELECT unnest(followings) FROM followers WHERE host='social.librem.one' AND acct = 'vincent' ) GROUP BY acct, id ORDER BY count DESC;
-```
-
-# Most active users
-
-```SQL
-SELECT array_length(followings, 1) AS l, acct, host FROM followers ORDER BY l DESC;
-```
-
-# Lag
-
-```SQL
-SELECT COUNT(*), date_trunc('day', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc;
-
-SELECT COUNT(*), date_trunc('hour', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc ORDER BY date_trunc;
-```
-
-Lag by shards: 
-
-```SQL
-SELECT min(lastsync), mod(id, 100) FROM
-    (SELECT acct, id, lastsync FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id) AS f
-GROUP BY mod
-ORDER BY min;
-```
-
-# Connections 
-
-```SQL
-SELECT SUM(cardinality(followings)) FROM followers;
-```

src/BSLManager/App.cs

@@ -0,0 +1,252 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Moderation.Actions;
+using BSLManager.Domain;
+using BSLManager.Tools;
+using Terminal.Gui;
+
+namespace BSLManager
+{
+    public class App
+    {
+        private readonly IFollowersDal _followersDal;
+        private readonly IRemoveFollowerAction _removeFollowerAction;
+
+        private readonly FollowersListState _state = new FollowersListState();
+
+        #region Ctor
+        public App(IFollowersDal followersDal, IRemoveFollowerAction removeFollowerAction)
+        {
+            _followersDal = followersDal;
+            _removeFollowerAction = removeFollowerAction;
+        }
+        #endregion
+
+        public void Run()
+        {
+            Application.Init();
+            var top = Application.Top;
+
+            // Creates the top-level window to show
+            var win = new Window("BSL Manager")
+            {
+                X = 0,
+                Y = 1, // Leave one row for the toplevel menu
+
+                // By using Dim.Fill(), it will automatically resize without manual intervention
+                Width = Dim.Fill(),
+                Height = Dim.Fill()
+            };
+
+            top.Add(win);
+
+            // Creates a menubar, the item "New" has a help menu.
+            var menu = new MenuBar(new MenuBarItem[]
+            {
+                new MenuBarItem("_File", new MenuItem[]
+                {
+                    new MenuItem("_Quit", "", () =>
+                    {
+                        if (Quit()) top.Running = false;
+                    })
+                }),
+                //new MenuBarItem ("_Edit", new MenuItem [] {
+                //    new MenuItem ("_Copy", "", null),
+                //    new MenuItem ("C_ut", "", null),
+                //    new MenuItem ("_Paste", "", null)
+                //})
+            });
+            top.Add(menu);
+
+            static bool Quit()
+            {
+                var n = MessageBox.Query(50, 7, "Quit BSL Manager", "Are you sure you want to quit?", "Yes", "No");
+                return n == 0;
+            }
+            
+            RetrieveUserList();
+
+            var list = new ListView(_state.GetDisplayableList())
+            {
+                X = 1,
+                Y = 3,
+                Width = Dim.Fill(),
+                Height = Dim.Fill()
+            };
+
+            list.KeyDown += _ =>
+            {
+                if (_.KeyEvent.Key == Key.Enter)
+                {
+                    OpenFollowerDialog(list.SelectedItem);
+                } 
+                else if (_.KeyEvent.Key == Key.Delete
+                           || _.KeyEvent.Key == Key.DeleteChar
+                           || _.KeyEvent.Key == Key.Backspace
+                           || _.KeyEvent.Key == Key.D)
+                {
+                    OpenDeleteDialog(list.SelectedItem);
+                }
+            };
+
+            var listingFollowersLabel = new Label(1, 0, "Listing followers");
+            var filterLabel = new Label("Filter: ") { X = 1, Y = 1 };
+            var filterText = new TextField("")
+            {
+                X = Pos.Right(filterLabel),
+                Y = 1,
+                Width = 40
+            };
+
+            filterText.KeyDown += _ =>
+            {
+                var text = filterText.Text.ToString();
+                if (_.KeyEvent.Key == Key.Enter && !string.IsNullOrWhiteSpace(text))
+                {
+                    _state.FilterBy(text);
+                    ConsoleGui.RefreshUI();
+                }
+            };
+
+            win.Add(
+                listingFollowersLabel,
+                filterLabel,
+                filterText,
+                list
+            );
+
+            Application.Run();
+        }
+
+        private void OpenFollowerDialog(int selectedIndex)
+        {
+            var close = new Button(3, 14, "Close");
+            close.Clicked += () => Application.RequestStop();
+
+            var dialog = new Dialog("Info", 60, 18, close);
+
+            var follower = _state.GetElementAt(selectedIndex);
+
+            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
+            {
+                X = 1,
+                Y = 1,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var following = new Label($"Following Count: {follower.Followings.Count}")
+            {
+                X = 1,
+                Y = 3,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var errors = new Label($"Posting Errors: {follower.PostingErrorCount}")
+            {
+                X = 1,
+                Y = 4,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var inbox = new Label($"Inbox: {follower.InboxRoute}")
+            {
+                X = 1,
+                Y = 5,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var sharedInbox = new Label($"Shared Inbox: {follower.SharedInboxRoute}")
+            {
+                X = 1,
+                Y = 6,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+
+            dialog.Add(name);
+            dialog.Add(following);
+            dialog.Add(errors);
+            dialog.Add(inbox);
+            dialog.Add(sharedInbox);
+            dialog.Add(close);
+            Application.Run(dialog);
+        }
+
+        private void OpenDeleteDialog(int selectedIndex)
+        {
+            bool okpressed = false;
+            var ok = new Button(10, 14, "Yes");
+            ok.Clicked += () =>
+            {
+                Application.RequestStop();
+                okpressed = true;
+            };
+
+            var cancel = new Button(3, 14, "No");
+            cancel.Clicked += () => Application.RequestStop();
+
+            var dialog = new Dialog("Delete", 60, 18, cancel, ok);
+
+            var follower = _state.GetElementAt(selectedIndex);
+            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
+            {
+                X = 1,
+                Y = 1,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var entry = new Label("Delete user and remove all their followings?")
+            {
+                X = 1,
+                Y = 3,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            dialog.Add(name);
+            dialog.Add(entry);
+            Application.Run(dialog);
+
+            if (okpressed)
+            {
+                DeleteAndRemoveUser(selectedIndex);
+            }
+        }
+
+        private void DeleteAndRemoveUser(int el)
+        {
+            Application.MainLoop.Invoke(async () =>
+            {
+                try
+                {
+                    var userToDelete = _state.GetElementAt(el);
+                    
+                    BasicLogger.Log($"Delete {userToDelete.Acct}@{userToDelete.Host}");
+                    await _removeFollowerAction.ProcessAsync(userToDelete);
+                    BasicLogger.Log($"Remove user from list");
+                    _state.RemoveAt(el);
+                }
+                catch (Exception e)
+                {
+                    BasicLogger.Log(e.Message);
+                }
+
+                ConsoleGui.RefreshUI();
+            });
+        }
+
+        private void RetrieveUserList()
+        {
+            Application.MainLoop.Invoke(async () =>
+            {
+                var followers = await _followersDal.GetAllFollowersAsync();
+                _state.Load(followers.ToList());
+                ConsoleGui.RefreshUI();
+            });
+        }
+    }
+}

src/BSLManager/BSLManager.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Lamar" Version="5.0.3" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
+    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
+    <ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
+    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="key.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/BSLManager/Bootstrapper.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Net.Http;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Common.Structs;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Postgres.DataAccessLayers;
+using BirdsiteLive.DAL.Postgres.Settings;
+using Lamar;
+using Lamar.Scanning.Conventions;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace BSLManager
+{
+    public class Bootstrapper
+    {
+        private readonly DbSettings _dbSettings;
+        private readonly InstanceSettings _instanceSettings;
+
+        #region Ctor
+        public Bootstrapper(DbSettings dbSettings, InstanceSettings instanceSettings)
+        {
+            _dbSettings = dbSettings;
+            _instanceSettings = instanceSettings;
+        }
+        #endregion
+
+        public Container Init()
+        {
+            var container = new Container(x =>
+            {
+                x.For<DbSettings>().Use(x => _dbSettings);
+
+                x.For<InstanceSettings>().Use(x => _instanceSettings);
+
+                if (string.Equals(_dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
+                {
+                    var connString = $"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
+                    var postgresSettings = new PostgresSettings
+                    {
+                        ConnString = connString
+                    };
+                    x.For<PostgresSettings>().Use(x => postgresSettings);
+
+                    x.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
+                    x.For<IFollowersDal>().Use<FollowersPostgresDal>().Singleton();
+                    x.For<IDbInitializerDal>().Use<DbInitializerPostgresDal>().Singleton();
+                }
+                else
+                {
+                    throw new NotImplementedException($"{_dbSettings.Type} is not supported");
+                }
+
+                var serviceProvider = new ServiceCollection().AddHttpClient().BuildServiceProvider();
+                x.For<IHttpClientFactory>().Use(_ => serviceProvider.GetService<IHttpClientFactory>());
+
+                x.For(typeof(ILogger<>)).Use(typeof(DummyLogger<>));
+
+                x.Scan(_ =>
+                {
+                    _.Assembly("BirdsiteLive.Twitter");
+                    _.Assembly("BirdsiteLive.Domain");
+                    _.Assembly("BirdsiteLive.DAL");
+                    _.Assembly("BirdsiteLive.DAL.Postgres");
+                    _.Assembly("BirdsiteLive.Moderation");
+
+                    _.TheCallingAssembly();
+
+                    _.WithDefaultConventions();
+
+                    _.LookForRegistries();
+                });
+            });
+            return container;
+        }
+
+        public class DummyLogger<T> : ILogger<T>
+        {
+            public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
+            {
+            }
+
+            public bool IsEnabled(LogLevel logLevel)
+            {
+                return false;
+            }
+
+            public IDisposable BeginScope<TState>(TState state)
+            {
+                return null;
+            }
+        }
+    }
+}

src/BSLManager/Domain/FollowersListState.cs

@@ -0,0 +1,81 @@
+using System.Collections.Generic;
+using System.Linq;
+using BirdsiteLive.DAL.Models;
+
+namespace BSLManager.Domain
+{
+    public class FollowersListState
+    {
+        private readonly List<string> _filteredDisplayableUserList = new List<string>();
+
+        private List<Follower> _sourceUserList = new List<Follower>();
+        private List<Follower> _filteredSourceUserList = new List<Follower>();
+        
+        public void Load(List<Follower> followers)
+        {
+            _sourceUserList = followers.OrderByDescending(x => x.Followings.Count).ToList();
+           
+            ResetLists();
+        }
+
+        private void ResetLists()
+        {
+            _filteredSourceUserList = _sourceUserList.ToList();
+
+            _filteredDisplayableUserList.Clear();
+
+            foreach (var follower in _sourceUserList)
+            {
+                var displayedUser = $"{GetFullHandle(follower)}   ({follower.Followings.Count})  (err:{follower.PostingErrorCount})";
+                _filteredDisplayableUserList.Add(displayedUser);
+            }
+        }
+
+        public List<string> GetDisplayableList()
+        {
+            return _filteredDisplayableUserList;
+        }
+
+        public void FilterBy(string pattern)
+        {
+            ResetLists();
+
+            if (!string.IsNullOrWhiteSpace(pattern))
+            {
+                var elToRemove = _filteredSourceUserList
+                    .Where(x => !GetFullHandle(x).Contains(pattern))
+                    .Select(x => x)
+                    .ToList();
+
+                foreach (var el in elToRemove)
+                {
+                    _filteredSourceUserList.Remove(el);
+                    
+                    var dElToRemove = _filteredDisplayableUserList.First(x => x.Contains(GetFullHandle(el)));
+                    _filteredDisplayableUserList.Remove(dElToRemove);
+                }
+            }
+        }
+
+        private string GetFullHandle(Follower follower)
+        {
+            return $"@{follower.Acct}@{follower.Host}";
+        }
+
+        public void RemoveAt(int index)
+        {
+            var displayableUser = _filteredDisplayableUserList[index];
+            var sourceUser = _filteredSourceUserList[index];
+
+            _filteredDisplayableUserList.Remove(displayableUser);
+
+            _filteredSourceUserList.Remove(sourceUser);
+            _sourceUserList.Remove(sourceUser);
+        }
+
+        public Follower GetElementAt(int index)
+        {
+            return _filteredSourceUserList[index];
+        }
+    }
+}

src/BSLManager/Program.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BSLManager.Tools;
+using Microsoft.Extensions.Configuration;
+using NStack;
+using Terminal.Gui;
+
+namespace BSLManager
+{
+    class Program
+    {
+        static async Task Main(string[] args)
+        {
+            Console.OutputEncoding = Encoding.Default;
+
+            var settingsManager = new SettingsManager();
+            var settings = settingsManager.GetSettings();
+
+            //var builder = new ConfigurationBuilder()
+            //    .AddEnvironmentVariables();
+            //var configuration = builder.Build();
+
+            //var dbSettings = configuration.GetSection("Db").Get<DbSettings>();
+            //var instanceSettings = configuration.GetSection("Instance").Get<InstanceSettings>();
+
+            var bootstrapper = new Bootstrapper(settings.dbSettings, settings.instanceSettings);
+            var container = bootstrapper.Init();
+
+            var app = container.GetInstance<App>();
+            app.Run();
+        }
+    }
+}

src/BSLManager/Tools/BasicLogger.cs

@@ -0,0 +1,13 @@
+using System;
+using System.IO;
+
+namespace BSLManager.Tools
+{
+    public static class BasicLogger
+    {
+        public static void Log(string log)
+        {
+            File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new []{ log });
+        }
+    }
+}

src/BSLManager/Tools/ConsoleGui.cs

@@ -0,0 +1,15 @@
+using System.Reflection;
+using Terminal.Gui;
+
+namespace BSLManager.Tools
+{
+    public static class ConsoleGui
+    {
+        public static void RefreshUI()
+        {
+            typeof(Application)
+                .GetMethod("TerminalResized", BindingFlags.Static | BindingFlags.NonPublic)
+                .Invoke(null, null);
+        }
+    }
+}

src/BSLManager/Tools/SettingsManager.cs

@@ -0,0 +1,124 @@
+using System;
+using System.IO;
+using System.Runtime.CompilerServices;
+using BirdsiteLive.Common.Settings;
+using Newtonsoft.Json;
+using Org.BouncyCastle.Asn1.IsisMtt.X509;
+
+namespace BSLManager.Tools
+{
+    public class SettingsManager
+    {
+        private const string LocalFileName = "ManagerSettings.json";
+
+        public (DbSettings dbSettings, InstanceSettings instanceSettings) GetSettings()
+        {
+            var localSettingsData = GetLocalSettingsFile();
+            if (localSettingsData != null) return Convert(localSettingsData);
+
+            Console.WriteLine("We need to set up the manager");
+            Console.WriteLine("Please provide the following information as provided in the docker-compose file");
+
+            LocalSettingsData data;
+            do
+            {
+                data = GetDataFromUser();
+                Console.WriteLine();
+                Console.WriteLine("Please check if all is ok:");
+                Console.WriteLine();
+                Console.WriteLine($"Db Host: {data.DbHost}");
+                Console.WriteLine($"Db Name: {data.DbName}");
+                Console.WriteLine($"Db User: {data.DbUser}");
+                Console.WriteLine($"Db Password: {data.DbPassword}");
+                Console.WriteLine($"Instance Domain: {data.InstanceDomain}");
+                Console.WriteLine();
+
+                string resp;
+                do
+                {
+                    Console.WriteLine("Is it valid? (yes, no)");
+                    resp = Console.ReadLine()?.Trim().ToLowerInvariant();
+
+                    if (resp == "n" || resp == "no") data = null;
+
+                } while (resp != "y" && resp != "yes" && resp != "n" && resp != "no");
+
+            } while (data == null);
+            
+            SaveLocalSettings(data);
+            return Convert(data);
+        }
+
+        private LocalSettingsData GetDataFromUser()
+        {
+            var data = new LocalSettingsData();
+
+            Console.WriteLine("Db Host:");
+            data.DbHost = Console.ReadLine();
+
+            Console.WriteLine("Db Name:");
+            data.DbName = Console.ReadLine();
+
+            Console.WriteLine("Db User:");
+            data.DbUser = Console.ReadLine();
+
+            Console.WriteLine("Db Password:");
+            data.DbPassword = Console.ReadLine();
+
+            Console.WriteLine("Instance Domain:");
+            data.InstanceDomain = Console.ReadLine();
+
+            return data;
+        }
+
+        private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(LocalSettingsData data)
+        {
+            var dbSettings = new DbSettings
+            {
+                Type = data.DbType,
+                Host = data.DbHost,
+                Name = data.DbName,
+                User = data.DbUser,
+                Password = data.DbPassword
+            };
+            var instancesSettings = new InstanceSettings
+            {
+                Domain = data.InstanceDomain
+            };
+            return (dbSettings, instancesSettings);
+        }
+
+        private LocalSettingsData GetLocalSettingsFile()
+        {
+            try
+            {
+                if (!File.Exists(LocalFileName)) return null;
+
+                var jsonContent = File.ReadAllText(LocalFileName);
+                var content = JsonConvert.DeserializeObject<LocalSettingsData>(jsonContent);
+                return content;
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+        }
+
+        private void SaveLocalSettings(LocalSettingsData data)
+        {
+            var jsonContent = JsonConvert.SerializeObject(data);
+            File.WriteAllText(LocalFileName, jsonContent);
+        }
+    }
+
+    internal class LocalSettingsData
+    {
+        public string DbType { get; set; } = "postgres";
+        public string DbHost { get; set; }
+        public string DbName { get; set; }
+        public string DbUser { get; set; }
+        public string DbPassword { get; set; }
+
+        public string InstanceDomain { get; set; }
+    }
+}

src/BirdsiteLive.ActivityPub/ApDeserializer.cs

@@ -1,7 +1,6 @@
 using System;
 using BirdsiteLive.ActivityPub.Models;
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
-using System.Text.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -11,21 +10,22 @@ namespace BirdsiteLive.ActivityPub
         {
             try
             {
-                var activity = JsonSerializer.Deserialize<Activity>(json);
+                var activity = JsonConvert.DeserializeObject<Activity>(json);
                 switch (activity.type)
                 {
                     case "Follow":
-                        return JsonSerializer.Deserialize<ActivityFollow>(json);
+                        return JsonConvert.DeserializeObject<ActivityFollow>(json);
                     case "Undo":
-                        var a = JsonSerializer.Deserialize<ActivityUndo>(json);
+                        var a = JsonConvert.DeserializeObject<ActivityUndo>(json);
                         if(a.apObject.type == "Follow")
-                            return JsonSerializer.Deserialize<ActivityUndoFollow>(json);
+                            return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
                         break;
                     case "Delete":
-                        return JsonSerializer.Deserialize<ActivityDelete>(json);
+                        return JsonConvert.DeserializeObject<ActivityDelete>(json);
                     case "Accept":
-                        var accept = JsonSerializer.Deserialize<ActivityAccept>(json);
+                        var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
-                        switch (accept.apObject.type)
+                        //var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);
+                        switch ((accept.apObject as dynamic).type.ToString())
                         {
                             case "Follow":
                                 var acceptFollow = new ActivityAcceptFollow()
@@ -36,12 +36,11 @@ namespace BirdsiteLive.ActivityPub
                                     context = accept.context,
                                     apObject = new ActivityFollow()
                                     {
-
+                                        id = (accept.apObject as dynamic).id?.ToString(),
-                                        id = accept.apObject.id,
+                                        type = (accept.apObject as dynamic).type?.ToString(),
-                                        type = accept.apObject.type,
+                                        actor = (accept.apObject as dynamic).actor?.ToString(),
-                                        actor = accept.apObject.actor,
+                                        context = (accept.apObject as dynamic).context?.ToString(),
-                                        context = accept.apObject.context?.ToString(),
+                                        apObject = (accept.apObject as dynamic).@object?.ToString()
-                                        apObject = accept.apObject.apObject,
                                     }
                                 };
                                 return acceptFollow;
@@ -57,5 +56,10 @@ namespace BirdsiteLive.ActivityPub
             return null;
         }
 
+        private class Ac : Activity
+        {
+            [JsonProperty("object")]
+            public Activity apObject { get; set; }
+        }
     }
 }

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -1,11 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="System.Text.Json" Version="4.7.2" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.ActivityPub/Converters/ContextArrayConverter.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace BirdsiteLive.ActivityPub.Converters
+{
+    public class ContextArrayConverter : JsonConverter
+    {
+        public override bool CanWrite { get { return false; } }
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            var result = new List<string>();
+
+            var list = serializer.Deserialize<List<object>>(reader);
+            foreach (var l in list)
+            {
+                if (l is string s)
+                    result.Add(s);
+                else
+                {
+                    var str = JsonConvert.SerializeObject(l);
+                    result.Add(str);
+                }
+            }
+
+            return result.ToArray();
+        }
+
+        public override bool CanConvert(Type objectType)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

src/BirdsiteLive.ActivityPub/Models/Activity.cs

@@ -1,16 +1,17 @@
 using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Activity
     {
-        [JsonPropertyName("@context")]
+        [JsonProperty("@context")]
-        public string context { get; set; }
+        public object context { get; set; }
         public string id { get; set; }
         public string type { get; set; }
         public string actor { get; set; }
 
-        //[JsonPropertyName("object")]
+        //[JsonProperty("object")]
         //public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAccept.cs

@@ -1,10 +1,10 @@
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAccept : Activity
     {
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
-        public NestedActivity apObject { get; set; }
+        public object apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptFollow.cs

@@ -1,10 +1,10 @@
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptFollow : Activity
     {
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityAcceptUndoFollow.cs

@@ -1,10 +1,10 @@
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityAcceptUndoFollow : Activity
     {
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
         public ActivityUndoFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityCreateNote.cs

@@ -1,5 +1,6 @@
-using BirdsiteLive.ActivityPub.Models;
+using System;
-using System.Text.Json.Serialization;
+using BirdsiteLive.ActivityPub.Models;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
@@ -9,7 +10,7 @@ namespace BirdsiteLive.ActivityPub
         public string[] to { get; set; }
         public string[] cc { get; set; }
 
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
         public Note apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs

@@ -1,10 +1,10 @@
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class ActivityDelete : Activity
     {
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
-        public string apObject { get; set; }
+        public object apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityFollow.cs

@@ -1,10 +1,10 @@
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityFollow : Activity
     {
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
         public string apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityRejectFollow.cs

@@ -1,10 +1,10 @@
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityRejectFollow : Activity
     {
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityUndo.cs

@@ -1,10 +1,10 @@
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndo : Activity
     {
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
         public Activity apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/ActivityUndoFollow.cs

@@ -1,10 +1,10 @@
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class ActivityUndoFollow : Activity
     {
-        [JsonPropertyName("object")]
+        [JsonProperty("object")]
         public ActivityFollow apObject { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/Actor.cs

@@ -1,13 +1,15 @@
 using System.Net;
 using BirdsiteLive.ActivityPub.Converters;
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
 {
     public class Actor
     {
-        [JsonPropertyName("@context")]
+        //[JsonPropertyName("@context")]
-        public object[] context { get; set; } = new string[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
+        [JsonProperty("@context")]
+        [JsonConverter(typeof(ContextArrayConverter))]
+        public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1" };
         public string id { get; set; }
         public string type { get; set; }
         public string followers { get; set; }

src/BirdsiteLive.ActivityPub/Models/Attachment.cs

@@ -1,14 +1,9 @@
-using System.Text.Json;
+namespace BirdsiteLive.ActivityPub
-using System.Text.Json.Serialization;
-
-namespace BirdsiteLive.ActivityPub
 {
     public class Attachment
     {
         public string type { get; set; }
         public string mediaType { get; set; }
         public string url { get; set; }
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public string name { get; set; }
     }
 }

src/BirdsiteLive.ActivityPub/Models/Followers.cs

@@ -1,11 +1,12 @@
 using BirdsiteLive.ActivityPub.Converters;
-using System.Text.Json.Serialization;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Followers
     {
-        [JsonPropertyName("@context")]
+        [JsonProperty("@context")]
+        [JsonConverter(typeof(ContextArrayConverter))]
         public string context { get; set; } = "https://www.w3.org/ns/activitystreams";
 
         public string id { get; set; }

src/BirdsiteLive.ActivityPub/Models/NestedActivity.cs

@@ -1,17 +0,0 @@
-using System.Text.Json.Serialization;
-
-namespace BirdsiteLive.ActivityPub
-{
-    public class NestedActivity
-    {
-        [JsonPropertyName("@context")]
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public object context { get; set; }
-        public string id { get; set; }
-        public string type { get; set; }
-        public string actor { get; set; }
-
-        [JsonPropertyName("object")]
-        public string apObject { get; set; }
-    }
-}

src/BirdsiteLive.ActivityPub/Models/Note.cs

@@ -1,10 +1,12 @@
-using System.Text.Json.Serialization;
+using BirdsiteLive.ActivityPub.Converters;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub.Models
 {
     public class Note
     {
-        [JsonPropertyName("@context")]
+        [JsonProperty("@context")]
+        [JsonConverter(typeof(ContextArrayConverter))]
         public string[] context { get; set; } = new[] { "https://www.w3.org/ns/activitystreams" };
 
         public string id { get; set; }

src/BirdsiteLive.Common/BirdsiteLive.Common.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
 </Project>

src/BirdsiteLive.Common/Regexes/HashtagRegexes.cs

@@ -5,6 +5,6 @@ namespace BirdsiteLive.Common.Regexes
     public class HashtagRegexes
     {
         public static readonly Regex HashtagName = new Regex(@"^[a-zA-Z0-9_]+$");
-        public static readonly Regex Hashtag = new Regex(@"(^|.?[ \n]+)#([a-zA-Z0-9_]+)(?=\s|$|[\[\]<>.,;:!?/|-])");
+        public static readonly Regex Hashtag = new Regex(@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])");
     }
 }

src/BirdsiteLive.Common/Regexes/UserRegexes.cs

@@ -5,6 +5,6 @@ namespace BirdsiteLive.Common.Regexes
     public class UserRegexes
     {
         public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
-        public static readonly Regex Mention = new Regex(@"(^|.?[ \n\.]+)@([a-zA-Z0-9_]+)(?=\s|$|[\[\]<>,;:'\.’!?/—\|-]|(. ))");
+        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:!?/|-]|(. ))");
     }
 }

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -7,6 +7,7 @@
         public string AdminEmail { get; set; }
         public bool ResolveMentionsInProfiles { get; set; }
         public bool PublishReplies { get; set; }
+        public int MaxUsersCapacity { get; set; }
 
         public string UnlistedTwitterAccounts { get; set; }
         public string SensitiveTwitterAccounts { get; set; }
@@ -14,14 +15,6 @@
         public int FailingTwitterUserCleanUpThreshold { get; set; }
         public int FailingFollowerCleanUpThreshold { get; set; } = -1;
 
-        public int UserCacheCapacity { get; set; } = 40_000;
+        public int UserCacheCapacity { get; set; }
-        public int TweetCacheCapacity { get; set; } = 20_000;
-        // "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
-        public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
-        public int m { get; set; } = 1;
-        public int n_start { get; set; } = 0;
-        public int n_end { get; set; } = 1;
-        public int ParallelTwitterRequests { get; set; } = 10;
-        public int ParallelFediversePosts { get; set; } = 10;
     }
 }

src/BirdsiteLive.Common/Settings/TwitterSettings.cs

@@ -0,0 +1,8 @@
+namespace BirdsiteLive.Common.Settings
+{
+    public class TwitterSettings
+    {
+        public string ConsumerKey { get; set; }
+        public string ConsumerSecret { get; set; }
+    }
+}

src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj

@@ -1,11 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Asn1" Version="1.0.9" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.8.6.7" />
   </ItemGroup>
 
 </Project>

src/BirdsiteLive.Cryptography/MagicKey.cs

@@ -1,12 +1,28 @@
 using System;
+using System.Collections.Generic;
 using System.Security.Cryptography;
 using System.Text;
-using System.Text.Json;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.Cryptography
 {
     public class MagicKey
     {
+        //public class WebfingerLink
+        //{
+        //    public string rel { get; set; }
+        //    public string type { get; set; }
+        //    public string href { get; set; }
+        //    public string template { get; set; }
+        //}
+
+        //public class WebfingerResult
+        //{
+        //    public string subject { get; set; }
+        //    public List<string> aliases { get; set; }
+        //    public List<WebfingerLink> links { get; set; }
+        //}
+
         private string[] _parts;
         private RSA _rsa;
 
@@ -22,14 +38,14 @@ namespace BirdsiteLive.Cryptography
 
         private class RSAKeyParms
         {
-            public byte[] D { get; set; }
+            public byte[] D;
-            public byte[] DP {get; set; }
+            public byte[] DP;
-            public byte[] DQ {get; set; }
+            public byte[] DQ;
-            public byte[] Exponent {get; set; }
+            public byte[] Exponent;
-            public byte[] InverseQ {get; set; }
+            public byte[] InverseQ;
-            public byte[] Modulus {get; set; }
+            public byte[] Modulus;
-            public byte[] P {get; set; }
+            public byte[] P;
-            public byte[] Q {get; set; }
+            public byte[] Q;
 
             public static RSAKeyParms From(RSAParameters parms)
             {
@@ -65,9 +81,7 @@ namespace BirdsiteLive.Cryptography
             if (key[0] == '{')
             {
                 _rsa = RSA.Create();
-                Console.WriteLine(key);
+                _rsa.ImportParameters(JsonConvert.DeserializeObject<RSAKeyParms>(key).Make());
-                Console.WriteLine(JsonSerializer.Deserialize<RSAKeyParms>(key).Make());
-                _rsa.ImportParameters(JsonSerializer.Deserialize<RSAKeyParms>(key).Make());
             }
             else
             {
@@ -88,7 +102,7 @@ namespace BirdsiteLive.Cryptography
             var rsa = RSA.Create();
             rsa.KeySize = 2048;
 
-            return new MagicKey(JsonSerializer.Serialize<RSAKeyParms>(RSAKeyParms.From(rsa.ExportParameters(true))));
+            return new MagicKey(JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true))));
         }
 
         public byte[] BuildSignedData(string data, string dataType, string encoding, string algorithm)
@@ -126,7 +140,7 @@ namespace BirdsiteLive.Cryptography
 
         public string PrivateKey
         {
-            get { return JsonSerializer.Serialize(RSAKeyParms.From(_rsa.ExportParameters(true))); }
+            get { return JsonConvert.SerializeObject(RSAKeyParms.From(_rsa.ExportParameters(true))); }
         }
 
         public string PublicKey

src/BirdsiteLive.Cryptography/RsaGenerator.cs

@@ -0,0 +1,99 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+
+namespace BirdsiteLive.Cryptography
+{
+    //https://gist.github.com/ststeiger/f4b29a140b1e3fd618679f89b7f3ff4a
+    //https://gist.github.com/valep27/4a720c25b35fff83fbf872516f847863
+    //https://gist.github.com/therightstuff/aa65356e95f8d0aae888e9f61aa29414
+    //https://stackoverflow.com/questions/52468125/export-rsa-public-key-in-der-format-and-decrypt-data
+    public class RsaGenerator
+    {
+        public string GetRsa()
+        {
+            var rsa = RSA.Create();
+            
+            var outputStream = new StringWriter();
+            var parameters = rsa.ExportParameters(true);
+            using (var stream = new MemoryStream())
+            {
+                var writer = new BinaryWriter(stream);
+                writer.Write((byte)0x30); // SEQUENCE
+                using (var innerStream = new MemoryStream())
+                {
+                    var innerWriter = new BinaryWriter(innerStream);
+                    innerWriter.Write((byte)0x30); // SEQUENCE
+                    EncodeLength(innerWriter, 13);
+                    innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
+                    var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
+                    EncodeLength(innerWriter, rsaEncryptionOid.Length);
+                    innerWriter.Write(rsaEncryptionOid);
+                    innerWriter.Write((byte)0x05); // NULL
+                    EncodeLength(innerWriter, 0);
+                    innerWriter.Write((byte)0x03); // BIT STRING
+                    using (var bitStringStream = new MemoryStream())
+                    {
+                        var bitStringWriter = new BinaryWriter(bitStringStream);
+                        bitStringWriter.Write((byte)0x00); // # of unused bits
+                        bitStringWriter.Write((byte)0x30); // SEQUENCE
+                        using (var paramsStream = new MemoryStream())
+                        {
+                            var paramsWriter = new BinaryWriter(paramsStream);
+                            //EncodeIntegerBigEndian(paramsWriter, parameters.Modulus); // Modulus
+                            //EncodeIntegerBigEndian(paramsWriter, parameters.Exponent); // Exponent
+                            var paramsLength = (int)paramsStream.Length;
+                            EncodeLength(bitStringWriter, paramsLength);
+                            bitStringWriter.Write(paramsStream.GetBuffer(), 0, paramsLength);
+                        }
+                        var bitStringLength = (int)bitStringStream.Length;
+                        EncodeLength(innerWriter, bitStringLength);
+                        innerWriter.Write(bitStringStream.GetBuffer(), 0, bitStringLength);
+                    }
+                    var length = (int)innerStream.Length;
+                    EncodeLength(writer, length);
+                    writer.Write(innerStream.GetBuffer(), 0, length);
+                }
+
+                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
+                // WriteLine terminates with \r\n, we want only \n
+                outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
+                for (var i = 0; i < base64.Length; i += 64)
+                {
+                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
+                    outputStream.Write("\n");
+                }
+                outputStream.Write("-----END PUBLIC KEY-----");
+            }
+
+            return outputStream.ToString();
+        }
+
+        private static void EncodeLength(BinaryWriter stream, int length)
+        {
+            if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
+            if (length < 0x80)
+            {
+                // Short form
+                stream.Write((byte)length);
+            }
+            else
+            {
+                // Long form
+                var temp = length;
+                var bytesRequired = 0;
+                while (temp > 0)
+                {
+                    temp >>= 8;
+                    bytesRequired++;
+                }
+                stream.Write((byte)(bytesRequired | 0x80));
+                for (var i = bytesRequired - 1; i >= 0; i--)
+                {
+                    stream.Write((byte)(length >> (8 * i) & 0xff));
+                }
+            }
+        }
+
+    }
+}

src/BirdsiteLive.Cryptography/RsaKeys.cs

@@ -0,0 +1,225 @@
+using Org.BouncyCastle.Crypto;
+using Org.BouncyCastle.Crypto.Parameters;
+using Org.BouncyCastle.OpenSsl;
+using Org.BouncyCastle.Security;
+using System;
+using System.IO;
+using System.Security.Cryptography;
+
+namespace MyProject.Data.Encryption
+{
+    public class RSAKeys
+    {
+        /// <summary>
+        /// Import OpenSSH PEM private key string into MS RSACryptoServiceProvider
+        /// </summary>
+        /// <param name="pem"></param>
+        /// <returns></returns>
+        public static RSACryptoServiceProvider ImportPrivateKey(string pem)
+        {
+            PemReader pr = new PemReader(new StringReader(pem));
+            AsymmetricCipherKeyPair KeyPair = (AsymmetricCipherKeyPair)pr.ReadObject();
+            RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaPrivateCrtKeyParameters)KeyPair.Private);
+
+            RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
+            csp.ImportParameters(rsaParams);
+            return csp;
+        }
+
+        /// <summary>
+        /// Import OpenSSH PEM public key string into MS RSACryptoServiceProvider
+        /// </summary>
+        /// <param name="pem"></param>
+        /// <returns></returns>
+        public static RSACryptoServiceProvider ImportPublicKey(string pem)
+        {
+            PemReader pr = new PemReader(new StringReader(pem));
+            AsymmetricKeyParameter publicKey = (AsymmetricKeyParameter)pr.ReadObject();
+            RSAParameters rsaParams = DotNetUtilities.ToRSAParameters((RsaKeyParameters)publicKey);
+
+            RSACryptoServiceProvider csp = new RSACryptoServiceProvider();// cspParams);
+            csp.ImportParameters(rsaParams);
+            return csp;
+        }
+
+        /// <summary>
+        /// Export private (including public) key from MS RSACryptoServiceProvider into OpenSSH PEM string
+        /// slightly modified from https://stackoverflow.com/a/23739932/2860309
+        /// </summary>
+        /// <param name="csp"></param>
+        /// <returns></returns>
+        public static string ExportPrivateKey(RSACryptoServiceProvider csp)
+        {
+            StringWriter outputStream = new StringWriter();
+            if (csp.PublicOnly) throw new ArgumentException("CSP does not contain a private key", "csp");
+            var parameters = csp.ExportParameters(true);
+            using (var stream = new MemoryStream())
+            {
+                var writer = new BinaryWriter(stream);
+                writer.Write((byte)0x30); // SEQUENCE
+                using (var innerStream = new MemoryStream())
+                {
+                    var innerWriter = new BinaryWriter(innerStream);
+                    EncodeIntegerBigEndian(innerWriter, new byte[] { 0x00 }); // Version
+                    EncodeIntegerBigEndian(innerWriter, parameters.Modulus);
+                    EncodeIntegerBigEndian(innerWriter, parameters.Exponent);
+                    EncodeIntegerBigEndian(innerWriter, parameters.D);
+                    EncodeIntegerBigEndian(innerWriter, parameters.P);
+                    EncodeIntegerBigEndian(innerWriter, parameters.Q);
+                    EncodeIntegerBigEndian(innerWriter, parameters.DP);
+                    EncodeIntegerBigEndian(innerWriter, parameters.DQ);
+                    EncodeIntegerBigEndian(innerWriter, parameters.InverseQ);
+                    var length = (int)innerStream.Length;
+                    EncodeLength(writer, length);
+                    writer.Write(innerStream.GetBuffer(), 0, length);
+                }
+
+                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
+                // WriteLine terminates with \r\n, we want only \n
+                outputStream.Write("-----BEGIN RSA PRIVATE KEY-----\n");
+                // Output as Base64 with lines chopped at 64 characters
+                for (var i = 0; i < base64.Length; i += 64)
+                {
+                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
+                    outputStream.Write("\n");
+                }
+                outputStream.Write("-----END RSA PRIVATE KEY-----");
+            }
+
+            return outputStream.ToString();
+        }
+
+        /// <summary>
+        /// Export public key from MS RSACryptoServiceProvider into OpenSSH PEM string
+        /// slightly modified from https://stackoverflow.com/a/28407693
+        /// </summary>
+        /// <param name="csp"></param>
+        /// <returns></returns>
+        public static string ExportPublicKey(RSACryptoServiceProvider csp)
+        {
+            StringWriter outputStream = new StringWriter();
+            var parameters = csp.ExportParameters(false);
+            using (var stream = new MemoryStream())
+            {
+                var writer = new BinaryWriter(stream);
+                writer.Write((byte)0x30); // SEQUENCE
+                using (var innerStream = new MemoryStream())
+                {
+                    var innerWriter = new BinaryWriter(innerStream);
+                    innerWriter.Write((byte)0x30); // SEQUENCE
+                    EncodeLength(innerWriter, 13);
+                    innerWriter.Write((byte)0x06); // OBJECT IDENTIFIER
+                    var rsaEncryptionOid = new byte[] { 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01 };
+                    EncodeLength(innerWriter, rsaEncryptionOid.Length);
+                    innerWriter.Write(rsaEncryptionOid);
+                    innerWriter.Write((byte)0x05); // NULL
+                    EncodeLength(innerWriter, 0);
+                    innerWriter.Write((byte)0x03); // BIT STRING
+                    using (var bitStringStream = new MemoryStream())
+                    {
+                        var bitStringWriter = new BinaryWriter(bitStringStream);
+                        bitStringWriter.Write((byte)0x00); // # of unused bits
+                        bitStringWriter.Write((byte)0x30); // SEQUENCE
+                        using (var paramsStream = new MemoryStream())
+                        {
+                            var paramsWriter = new BinaryWriter(paramsStream);
+                            EncodeIntegerBigEndian(paramsWriter, parameters.Modulus); // Modulus
+                            EncodeIntegerBigEndian(paramsWriter, parameters.Exponent); // Exponent
+                            var paramsLength = (int)paramsStream.Length;
+                            EncodeLength(bitStringWriter, paramsLength);
+                            bitStringWriter.Write(paramsStream.GetBuffer(), 0, paramsLength);
+                        }
+                        var bitStringLength = (int)bitStringStream.Length;
+                        EncodeLength(innerWriter, bitStringLength);
+                        innerWriter.Write(bitStringStream.GetBuffer(), 0, bitStringLength);
+                    }
+                    var length = (int)innerStream.Length;
+                    EncodeLength(writer, length);
+                    writer.Write(innerStream.GetBuffer(), 0, length);
+                }
+
+                var base64 = Convert.ToBase64String(stream.GetBuffer(), 0, (int)stream.Length).ToCharArray();
+                // WriteLine terminates with \r\n, we want only \n
+                outputStream.Write("-----BEGIN PUBLIC KEY-----\n");
+                for (var i = 0; i < base64.Length; i += 64)
+                {
+                    outputStream.Write(base64, i, Math.Min(64, base64.Length - i));
+                    outputStream.Write("\n");
+                }
+                outputStream.Write("-----END PUBLIC KEY-----");
+            }
+
+            return outputStream.ToString();
+        }
+
+        /// <summary>
+        /// https://stackoverflow.com/a/23739932/2860309
+        /// </summary>
+        /// <param name="stream"></param>
+        /// <param name="length"></param>
+        private static void EncodeLength(BinaryWriter stream, int length)
+        {
+            if (length < 0) throw new ArgumentOutOfRangeException("length", "Length must be non-negative");
+            if (length < 0x80)
+            {
+                // Short form
+                stream.Write((byte)length);
+            }
+            else
+            {
+                // Long form
+                var temp = length;
+                var bytesRequired = 0;
+                while (temp > 0)
+                {
+                    temp >>= 8;
+                    bytesRequired++;
+                }
+                stream.Write((byte)(bytesRequired | 0x80));
+                for (var i = bytesRequired - 1; i >= 0; i--)
+                {
+                    stream.Write((byte)(length >> (8 * i) & 0xff));
+                }
+            }
+        }
+
+        /// <summary>
+        /// https://stackoverflow.com/a/23739932/2860309
+        /// </summary>
+        /// <param name="stream"></param>
+        /// <param name="value"></param>
+        /// <param name="forceUnsigned"></param>
+        private static void EncodeIntegerBigEndian(BinaryWriter stream, byte[] value, bool forceUnsigned = true)
+        {
+            stream.Write((byte)0x02); // INTEGER
+            var prefixZeros = 0;
+            for (var i = 0; i < value.Length; i++)
+            {
+                if (value[i] != 0) break;
+                prefixZeros++;
+            }
+            if (value.Length - prefixZeros == 0)
+            {
+                EncodeLength(stream, 1);
+                stream.Write((byte)0);
+            }
+            else
+            {
+                if (forceUnsigned && value[prefixZeros] > 0x7f)
+                {
+                    // Add a prefix zero to force unsigned if the MSB is 1
+                    EncodeLength(stream, value.Length - prefixZeros + 1);
+                    stream.Write((byte)0);
+                }
+                else
+                {
+                    EncodeLength(stream, value.Length - prefixZeros);
+                }
+                for (var i = prefixZeros; i < value.Length; i++)
+                {
+                    stream.Write(value[i]);
+                }
+            }
+        }
+    }
+}

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -4,14 +4,13 @@ using System.Net;
 using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.Domain
 {
@@ -19,10 +18,8 @@ namespace BirdsiteLive.Domain
     {
         Task<Actor> GetUser(string objectId);
         Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
-        Task PostNewActivity(ActivityCreateNote note, string username, string noteId, string targetHost,
+        Task PostNewActivity(Note note, string username, string activityType, string noteId, string targetHost,
             string targetInbox);
-
-        ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity);
     }
 
     public class ActivityPubService : IActivityPubService
@@ -55,16 +52,40 @@ namespace BirdsiteLive.Domain
 
             var content = await result.Content.ReadAsStringAsync();
 
-            var actor = JsonSerializer.Deserialize<Actor>(content);
+            var actor = JsonConvert.DeserializeObject<Actor>(content);
             if (string.IsNullOrWhiteSpace(actor.url)) actor.url = objectId;
             return actor;
         }
 
-        public async Task PostNewActivity(ActivityCreateNote noteActivity, string username, string noteId, string targetHost, string targetInbox)
+        public async Task PostNewActivity(Note note, string username, string activityType, string noteId, string targetHost, string targetInbox)
         {
             try
             {
                 var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
+                String noteUri;
+                if (activityType == "Create") 
+                {
+                    noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, noteId);
+                } else
+                {
+                    noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, note.announceId);
+                }
+
+                var now = DateTime.UtcNow;
+                var nowString = now.ToString("s") + "Z";
+
+                var noteActivity = new ActivityCreateNote()
+                {
+                    context = "https://www.w3.org/ns/activitystreams",
+                    id = $"{noteUri}/activity",
+                    type = activityType,
+                    actor = actor,
+                    published = nowString,
+
+                    to = new[] {$"{actor}/followers"},
+                    cc = note.cc,
+                    apObject = note
+                };
 
                 await PostDataAsync(noteActivity, targetHost, actor, targetInbox);
             }
@@ -75,32 +96,13 @@ namespace BirdsiteLive.Domain
             }
         }
 
-        public ActivityAcceptFollow BuildAcceptFollow(ActivityFollow activity)
+        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
-        {
-            var acceptFollow = new ActivityAcceptFollow()
-            {
-                context = "https://www.w3.org/ns/activitystreams",
-                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
-                type = "Accept",
-                actor = activity.apObject,
-                apObject = new ActivityFollow()
-                {
-                    id = activity.id,
-                    type = activity.type,
-                    actor = activity.actor,
-                    apObject = activity.apObject
-                }
-            };
-            return acceptFollow;
-        }
-        public HttpRequestMessage BuildRequest<T>(T data, string targetHost, string actorUrl,
-            string inbox = null)
         {
             var usedInbox = $"/inbox";
             if (!string.IsNullOrWhiteSpace(inbox))
                 usedInbox = inbox;
 
-            var json = JsonSerializer.Serialize(data);
+            var json = JsonConvert.SerializeObject(data, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
 
             var date = DateTime.UtcNow.ToUniversalTime();
             var httpDate = date.ToString("r");
@@ -109,32 +111,28 @@ namespace BirdsiteLive.Domain
 
             var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
 
+            var client = _httpClientFactory.CreateClient();
             var httpRequestMessage = new HttpRequestMessage
             {
                 Method = HttpMethod.Post,
                 RequestUri = new Uri($"https://{targetHost}{usedInbox}"),
                 Headers =
                 {
-                    { "Host", targetHost },
+                    {"Host", targetHost},
-                    { "Date", httpDate },
+                    {"Date", httpDate},
-                    { "Signature", signature },
+                    {"Signature", signature},
-                    { "Digest", $"SHA-256={digest}" }
+                    {"Digest", $"SHA-256={digest}"}
                 },
                 Content = new StringContent(json, Encoding.UTF8, "application/ld+json")
             };
 
-            return httpRequestMessage;
-        }
-
-        public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
-        {
-            var httpRequestMessage = BuildRequest(data, targetHost, actorUrl, inbox);
-
-            var client = _httpClientFactory.CreateClient();
-            client.Timeout = TimeSpan.FromSeconds(2);
-            
             var response = await client.SendAsync(httpRequestMessage);
             response.EnsureSuccessStatusCode();
+            _logger.LogInformation("Sent tweet to " + targetHost);
+            _logger.LogInformation("Tweet content is " + json);
+
+            var c = await response.Content.ReadAsStringAsync();
+            _logger.LogInformation("Got res after posting tweet " + c);
 
             return response.StatusCode;
         }

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs

@@ -42,6 +42,9 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             var twitterUserId = twitterUser.Id;
             if(!follower.Followings.Contains(twitterUserId))
                 follower.Followings.Add(twitterUserId);
+
+            if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
+                follower.FollowingsSyncStatus.Add(twitterUserId, -1);
             
             // Save Follower
             await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs

@@ -36,6 +36,9 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             if (follower.Followings.Contains(twitterUserId))
                 follower.Followings.Remove(twitterUserId);
 
+            if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
+                follower.FollowingsSyncStatus.Remove(twitterUserId);
+
             // Save or delete Follower
             if (follower.Followings.Any())
                 await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/Repository/PublicationRepository.cs

@@ -0,0 +1,40 @@
+using System.Linq;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Domain.Tools;
+
+namespace BirdsiteLive.Domain.Repository
+{
+    public interface IPublicationRepository
+    {
+        bool IsUnlisted(string twitterAcct);
+        bool IsSensitive(string twitterAcct);
+    }
+
+    public class PublicationRepository : IPublicationRepository
+    {
+        private readonly string[] _unlistedAccounts;
+        private readonly string[] _sensitiveAccounts;
+
+        #region Ctor
+        public PublicationRepository(InstanceSettings settings)
+        {
+            _unlistedAccounts = PatternsParser.Parse(settings.UnlistedTwitterAccounts);
+            _sensitiveAccounts = PatternsParser.Parse(settings.SensitiveTwitterAccounts);
+        }
+        #endregion
+
+        public bool IsUnlisted(string twitterAcct)
+        {
+            if (_unlistedAccounts == null || !_unlistedAccounts.Any()) return false;
+
+            return _unlistedAccounts.Contains(twitterAcct.ToLowerInvariant());
+        }
+
+        public bool IsSensitive(string twitterAcct)
+        {
+            if (_sensitiveAccounts == null || !_sensitiveAccounts.Any()) return false;
+
+            return _sensitiveAccounts.Contains(twitterAcct.ToLowerInvariant());
+        }
+    }
+}

src/BirdsiteLive.Domain/StatusService.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
-using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
@@ -12,13 +11,14 @@ using BirdsiteLive.Domain.Repository;
 using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
+using Tweetinvi.Models;
+using Tweetinvi.Models.Entities;
 
 namespace BirdsiteLive.Domain
 {
     public interface IStatusService
     {
         Note GetStatus(string username, ExtractedTweet tweet);
-        ActivityCreateNote GetActivity(string username, ExtractedTweet tweet);
     }
 
     public class StatusService : IStatusService
@@ -26,13 +26,15 @@ namespace BirdsiteLive.Domain
         private readonly InstanceSettings _instanceSettings;
         private readonly IStatusExtractor _statusExtractor;
         private readonly IExtractionStatisticsHandler _statisticsHandler;
+        private readonly IPublicationRepository _publicationRepository;
 
         #region Ctor
-        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
+        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, IPublicationRepository publicationRepository)
         {
             _instanceSettings = instanceSettings;
             _statusExtractor = statusExtractor;
             _statisticsHandler = statisticsHandler;
+            _publicationRepository = publicationRepository;
         }
         #endregion
 
@@ -44,15 +46,21 @@ namespace BirdsiteLive.Domain
             if (tweet.IsRetweet)
             {
                 actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Acct);
-                noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Acct, tweet.RetweetId.ToString());
+                noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, tweet.OriginalAuthor.Acct, tweet.Id.ToString());
-                announceId  = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
+                announceId  = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.RetweetId.ToString());
             }
 
             var to = $"{actorUrl}/followers";
 
+            var isUnlisted = _publicationRepository.IsUnlisted(username);
             var cc = new string[0];
+            if (isUnlisted)
+                cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
             
             string summary = null;
+            var sensitive = _publicationRepository.IsSensitive(username);
+            if (sensitive)
+                summary = "Potential Content Warning";
 
             var extractedTags = _statusExtractor.Extract(tweet.MessageContent);
             _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
@@ -67,8 +75,8 @@ namespace BirdsiteLive.Domain
             cc = new[] {"https://www.w3.org/ns/activitystreams#Public"};
 
             string inReplyTo = null;
-            if (tweet.InReplyToStatusId != default)
+         //   if (tweet.InReplyToStatusId != default)
-                inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
+         //       inReplyTo = $"https://{_instanceSettings.Domain}/users/{tweet.InReplyToAccount.ToLowerInvariant()}/statuses/{tweet.InReplyToStatusId}";
 
             var note = new Note
             {
@@ -84,7 +92,7 @@ namespace BirdsiteLive.Domain
                 to = new[] { to },
                 cc = cc,
 
-                sensitive = false,
+                sensitive = sensitive,
                 summary = summary,
                 content = $"<p>{content}</p>",
                 attachment = Convert(tweet.Media),
@@ -93,40 +101,6 @@ namespace BirdsiteLive.Domain
 
             return note;
         }
-        public ActivityCreateNote GetActivity(string username, ExtractedTweet tweet)
-        {
-            var note = GetStatus(username, tweet);
-            var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
-            String noteUri;
-            string activityType;
-            if (tweet.IsRetweet) 
-            {
-                noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
-                activityType = "Announce";
-            } else
-            {
-                noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, tweet.Id.ToString());
-                activityType = "Create";
-            }
-
-            var now = DateTime.UtcNow;
-            var nowString = now.ToString("s") + "Z";
-
-            var noteActivity = new ActivityCreateNote()
-            {
-                context = "https://www.w3.org/ns/activitystreams",
-                id = $"{noteUri}/activity",
-                type = activityType,
-                actor = actor,
-                published = nowString,
-
-                to = new[] {$"{actor}/followers"},
-                cc = note.cc,
-                apObject = note
-            };
-
-            return noteActivity;
-        }
 
         private Attachment[] Convert(ExtractedMedia[] media)
         {
@@ -137,8 +111,7 @@ namespace BirdsiteLive.Domain
                 {
                     type = "Document",
                     url = x.Url,
-                    mediaType = x.MediaType,
+                    mediaType = x.MediaType
-                    name = x.AltText
                 };
             }).ToArray();
         }

src/BirdsiteLive.Domain/Tools/ModerationRegexParser.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Text.RegularExpressions;
 using BirdsiteLive.Domain.Repository;
+using Org.BouncyCastle.Pkcs;
 
 namespace BirdsiteLive.Domain.Tools
 {

src/BirdsiteLive.Domain/Tools/StatusExtractor.cs

@@ -6,7 +6,6 @@ using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Twitter;
 using Microsoft.Extensions.Logging;
-using System;
 
 namespace BirdsiteLive.Domain.Tools
 {
@@ -32,7 +31,14 @@ namespace BirdsiteLive.Domain.Tools
         {
             var tags = new List<Tag>();
 
+            // Replace return lines
+            messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p>");
+            messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/>");
 
+            //// Secure emojis
+            //var emojiMatch = EmojiRegexes.Emoji.Matches(messageContent);
+            //foreach (Match m in emojiMatch)
+            //    messageContent = Regex.Replace(messageContent, m.ToString(), $" {m} ");
 
             // Extract Urls
             var urlMatch = UrlRegexes.Url.Matches(messageContent);
@@ -104,8 +110,8 @@ namespace BirdsiteLive.Domain.Tools
                         continue;
                     }
 
-                    var url = $"https://{_instanceSettings.Domain}/users/{mention.ToLower()}";
+                    var url = $"https://{_instanceSettings.Domain}/users/{mention}";
-                    var name = $"@{mention.ToLower()}";
+                    var name = $"@{mention}@{_instanceSettings.Domain}";
 
                     if (tags.All(x => x.href != url))
                     {
@@ -118,13 +124,9 @@ namespace BirdsiteLive.Domain.Tools
                     }
 
                     messageContent = Regex.Replace(messageContent, Regex.Escape(m.Groups[0].ToString()),
-                        $@"{m.Groups[1]}<span class=""h-card""><a href=""{url}"" class=""u-url mention"">@<span>{mention.ToLower()}</span></a></span>{m.Groups[3]}");
+                        $@"{m.Groups[1]}<span class=""h-card""><a href=""https://{_instanceSettings.Domain}/@{mention}"" class=""u-url mention"">@<span>{mention}</span></a></span>{m.Groups[3]}");
                 }
             }
-            
-            // Replace return lines
-            messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p>");
-            messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/>");
 
             return (messageContent.Trim(), tags.ToArray());
         }

src/BirdsiteLive.Domain/UserService.cs

@@ -17,6 +17,8 @@ using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
+using Tweetinvi.Core.Exceptions;
+using Tweetinvi.Models;
 
 namespace BirdsiteLive.Domain
 {
@@ -85,7 +87,7 @@ namespace BirdsiteLive.Domain
                 preferredUsername = acct,
                 name = twitterUser.Name,
                 inbox = $"{actorUrl}/inbox",
-                summary = "This account is a replica from Twitter. Its author can't see your replies. If you find this service useful, please consider supporting us via our Patreon. <br>" + description,
+                summary = description,
                 url = actorUrl,
                 manuallyApprovesFollowers = twitterUser.Protected,
                 publicKey = new PublicKey()
@@ -111,12 +113,6 @@ namespace BirdsiteLive.Domain
                         type = "PropertyValue",
                         name = "Official",
                         value = $"<a href=\"https://twitter.com/{acct}\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">twitter.com/{acct}</span></a>"
-                    },
-                    new UserAttachment
-                    {
-                        type = "PropertyValue",
-                        name = "Support this service",
-                        value = $"<a href=\"https://www.patreon.com/birddotmakeup\" rel=\"me nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">www.patreon.com/birddotmakeup</span></a>"
                     }
                 },
                 endpoints = new EndPoints
@@ -166,7 +162,7 @@ namespace BirdsiteLive.Domain
             }
 
             // Validate User Protected
-            var user = await _twitterUserService.GetUserAsync(twitterUser);
+            var user = _twitterUserService.GetUser(twitterUser);
             if (!user.Protected)
             {
                 // Execute
@@ -182,11 +178,23 @@ namespace BirdsiteLive.Domain
         
         private async Task<bool> SendAcceptFollowAsync(ActivityFollow activity, string followerHost)
         {
-            var acceptFollow = _activityPubService.BuildAcceptFollow(activity);
+            var acceptFollow = new ActivityAcceptFollow()
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{activity.apObject}#accepts/follows/{Guid.NewGuid()}",
+                type = "Accept",
+                actor = activity.apObject,
+                apObject = new ActivityFollow()
+                {
+                    id = activity.id,
+                    type = activity.type,
+                    actor = activity.actor,
+                    apObject = activity.apObject
+                }
+            };
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
             return result == HttpStatusCode.Accepted ||
                    result == HttpStatusCode.OK; //TODO: revamp this for better error handling
-
         }
 
         public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost)
@@ -244,11 +252,10 @@ namespace BirdsiteLive.Domain
                 actor = activity.apObject.apObject,
                 apObject = new ActivityUndoFollow()
                 {
-                    id = (activity.apObject as dynamic).id?.ToString(),
+                    id = activity.id,
-                    type = (activity.apObject as dynamic).type?.ToString(),
+                    type = activity.type,
-                    actor = (activity.apObject as dynamic).actor?.ToString(),
+                    actor = activity.actor,
-                    context = (activity.apObject as dynamic).context?.ToString(),
+                    apObject = activity.apObject
-                    apObject = (activity.apObject as dynamic).@object?.ToString()
                 }
             };
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);

src/BirdsiteLive.Moderation/Actions/RemoveTwitterAccountAction.cs

@@ -41,6 +41,9 @@ namespace BirdsiteLive.Moderation.Actions
                 if (follower.Followings.Contains(twitterUserId))
                     follower.Followings.Remove(twitterUserId);
 
+                if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
+                    follower.FollowingsSyncStatus.Remove(twitterUserId);
+
                 if (follower.Followings.Any())
                     await _followersDal.UpdateFollowerAsync(follower);
                 else

src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
-    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="6.0" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/Contracts/IRefreshTwitterUserStatusProcessor.cs

@@ -0,0 +1,12 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface IRefreshTwitterUserStatusProcessor
+    {
+        Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/IRetrieveTwitterUsersProcessor.cs

@@ -1,12 +1,12 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
-using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.DAL.Models;
 
 namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface IRetrieveTwitterUsersProcessor
     {
-        Task GetTwitterUsersAsync(BufferBlock<UserWithDataToSync[]> twitterUsersBufferBlock, CancellationToken ct);
+        Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionProcessor.cs

@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface ISaveProgressionProcessor
+    {
+        Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Contracts/ISendTweetsToFollowersProcessor.cs

@@ -6,6 +6,6 @@ namespace BirdsiteLive.Pipeline.Contracts
 {
     public interface ISendTweetsToFollowersProcessor
     {
-        Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct);
+        Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
     }
 }

src/BirdsiteLive.Pipeline/Models/UserWithDataToSync.cs

@@ -1,5 +1,6 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Twitter.Models;
+using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Models
 {

src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Moderation.Actions;
+using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Twitter;
+using BirdsiteLive.Twitter.Models;
+
+namespace BirdsiteLive.Pipeline.Processors
+{
+    public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
+    {
+        private readonly ICachedTwitterUserService _twitterUserService;
+        private readonly ITwitterUserDal _twitterUserDal;
+        private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction;
+        private readonly InstanceSettings _instanceSettings;
+
+        #region Ctor
+        public RefreshTwitterUserStatusProcessor(ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, IRemoveTwitterAccountAction removeTwitterAccountAction, InstanceSettings instanceSettings)
+        {
+            _twitterUserService = twitterUserService;
+            _twitterUserDal = twitterUserDal;
+            _removeTwitterAccountAction = removeTwitterAccountAction;
+            _instanceSettings = instanceSettings;
+        }
+        #endregion
+
+        public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitterUsers, CancellationToken ct)
+        {
+            var usersWtData = new List<UserWithDataToSync>();
+
+            foreach (var user in syncTwitterUsers)
+            {
+                TwitterUser userView = null;
+
+                try
+                {
+                    userView = _twitterUserService.GetUser(user.Acct);
+                }
+                catch (UserNotFoundException)
+                {
+                    await ProcessNotFoundUserAsync(user);
+                    continue;
+                }
+                catch (UserHasBeenSuspendedException)
+                {
+                    await ProcessNotFoundUserAsync(user);
+                    continue;
+                }
+                catch (RateLimitExceededException)
+                {
+                    await ProcessRateLimitExceededAsync(user);
+                    continue;
+                }
+                catch (Exception)
+                {
+                    // ignored
+                }
+
+                if (userView == null || userView.Protected)
+                {
+                    await ProcessFailingUserAsync(user);
+                    continue;
+                }
+
+                user.FetchingErrorCount = 0;
+                var userWtData = new UserWithDataToSync
+                {
+                    User = user
+                };
+                usersWtData.Add(userWtData);
+            }
+            return usersWtData.ToArray();
+        }
+
+        private async Task ProcessRateLimitExceededAsync(SyncTwitterUser user)
+        {
+            var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
+            dbUser.LastSync = DateTime.UtcNow;
+            await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
+        }
+
+        private async Task ProcessNotFoundUserAsync(SyncTwitterUser user)
+        {
+            await _removeTwitterAccountAction.ProcessAsync(user);
+        }
+
+        private async Task ProcessFailingUserAsync(SyncTwitterUser user)
+        {
+            var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
+            dbUser.FetchingErrorCount++;
+            dbUser.LastSync = DateTime.UtcNow;
+
+            if (dbUser.FetchingErrorCount > _instanceSettings.FailingTwitterUserCleanUpThreshold)
+            {
+                await _removeTwitterAccountAction.ProcessAsync(user);
+            }
+            else
+            {
+                await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
+            }
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
@@ -21,18 +20,12 @@ namespace BirdsiteLive.Pipeline.Processors
 
         public async Task<IEnumerable<UserWithDataToSync>> ProcessAsync(UserWithDataToSync[] userWithTweetsToSyncs, CancellationToken ct)
         {
-            //List<Task> todo = new List<Task>();
+            //TODO multithread this
-            //foreach (var user in userWithTweetsToSyncs)
+            foreach (var user in userWithTweetsToSyncs)
-            //{
+            {
-            //    var t = Task.Run( 
+                var followers = await _followersDal.GetFollowersAsync(user.User.Id);
-            //    async() => {
+                user.Followers = followers;
-            //        var followers = await _followersDal.GetFollowersAsync(user.User.Id);
+            }
-            //        user.Followers = followers;
-            //    });
-            //    todo.Add(t);
-            //}
-            //
-            //await Task.WhenAll(todo);
 
             return userWithTweetsToSyncs;
         }

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -10,10 +9,10 @@ using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
-using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
+using Tweetinvi.Models;
 
-namespace BirdsiteLive.Pipeline.Processors.SubTasks
+namespace BirdsiteLive.Pipeline.Processors
 {
     public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
     {
@@ -21,90 +20,58 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         private readonly ICachedTwitterUserService _twitterUserService;
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<RetrieveTweetsProcessor> _logger;
-        private readonly InstanceSettings _settings;
 
         #region Ctor
-        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, InstanceSettings settings, ILogger<RetrieveTweetsProcessor> logger)
+        public RetrieveTweetsProcessor(ITwitterTweetsService twitterTweetsService, ITwitterUserDal twitterUserDal, ICachedTwitterUserService twitterUserService, ILogger<RetrieveTweetsProcessor> logger)
         {
             _twitterTweetsService = twitterTweetsService;
             _twitterUserDal = twitterUserDal;
             _twitterUserService = twitterUserService;
             _logger = logger;
-            _settings = settings;
         }
         #endregion
 
         public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTwitterUsers, CancellationToken ct)
         {
+            var usersWtTweets = new List<UserWithDataToSync>();
 
-            if (_settings.ParallelTwitterRequests == 0)
+            //TODO multithread this
-            {
-                while(true)
-                    await Task.Delay(1000);
-            }
-
-            var usersWtTweets = new ConcurrentBag<UserWithDataToSync>();
-            List<Task> todo = new List<Task>();
-            int index = 0;
             foreach (var userWtData in syncTwitterUsers)
             {
-                index++;
+                var user = userWtData.User;
-
+                var tweets = RetrieveNewTweets(user);
-                var t = Task.Run(async () => {
+                _logger.LogInformation("Got " + tweets.Length + " tweets from user " + user.Acct);
-                    var user = userWtData.User;
+                if (tweets.Length > 0 && user.LastTweetPostedId != -1)
-                    var now = DateTime.UtcNow;
-                    try 
-                    {
-                        var tweets = await RetrieveNewTweets(user);
-                        _logger.LogInformation(index + "/" + syncTwitterUsers.Count() + " Got " + tweets.Length + " tweets from user " + user.Acct + " " );
-                        if (tweets.Length > 0 && user.LastTweetPostedId == -1)
-                        {
-                            // skip the first time to avoid sending backlog of tweet
-                            var tweetId = tweets.Last().Id;
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
-                        }
-                        else if (tweets.Length > 0 && user.LastTweetPostedId != -1)
-                        {
-                            userWtData.Tweets = tweets;
-                            usersWtTweets.Add(userWtData);
-                            var tweetId = tweets.Last().Id;
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
-                        }
-                        else
-                        {
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
-                        }
-
-                    } 
-                    catch(Exception e)
-                    {
-                        _logger.LogError(e.Message);
-                        await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
-                    }
-                });
-                todo.Add(t);
-                if (todo.Count > _settings.ParallelTwitterRequests)
                 {
-                    await Task.WhenAll(todo);
+                    userWtData.Tweets = tweets;
-                    todo.Clear();
+                    usersWtTweets.Add(userWtData);
+                }
+                else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
+                {
+                    var tweetId = tweets.Last().Id;
+                    var now = DateTime.UtcNow;
+                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
+                }
+                else
+                {
+                    var now = DateTime.UtcNow;
+                    await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
                 }
-                
             }
 
-            await Task.WhenAll(todo);
             return usersWtTweets.ToArray();
         }
 
-        private async Task<ExtractedTweet[]> RetrieveNewTweets(SyncTwitterUser user)
+        private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
         {
             var tweets = new ExtractedTweet[0];
             
             try
             {
                 if (user.LastTweetPostedId == -1)
-                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct);
+                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
                 else
-                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetPostedId);
+                    tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
             }
             catch (Exception e)
             {

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -7,8 +6,9 @@ using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.Common.Extensions;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Tools;
 using Microsoft.Extensions.Logging;
 
 namespace BirdsiteLive.Pipeline.Processors
@@ -16,61 +16,57 @@ namespace BirdsiteLive.Pipeline.Processors
     public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
-        private readonly IFollowersDal _followersDal;
+        private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
-        private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
-        private static Random rng = new Random();
         
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
-            _followersDal = followersDal;
+            _maxUsersNumberProvider = maxUsersNumberProvider;
-            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
 
-        public async Task GetTwitterUsersAsync(BufferBlock<UserWithDataToSync[]> twitterUsersBufferBlock, CancellationToken ct)
+        public async Task GetTwitterUsersAsync(BufferBlock<SyncTwitterUser[]> twitterUsersBufferBlock, CancellationToken ct)
         {
             for (; ; )
             {
                 ct.ThrowIfCancellationRequested();
 
-                if (_instanceSettings.ParallelTwitterRequests == 0)
+                try
                 {
-                    while (true)
+                    var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
-                        await Task.Delay(10000);
+                    var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
-                }
-                
-                var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
 
-                var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
+                    var userCount = users.Any() ? users.Length : 1;
-                var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
+                    var splitNumber = (int) Math.Ceiling(userCount / 15d);
+                    var splitUsers = users.Split(splitNumber).ToList();
 
-                foreach (var users in splitUsers)
+                    foreach (var u in splitUsers)
-                {
-                    ct.ThrowIfCancellationRequested();
-                    List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
-                    foreach (var u in users)
                     {
-                        var followers = await _followersDal.GetFollowersAsync(u.Id);
+                        ct.ThrowIfCancellationRequested();
-                        toSync.Add( new UserWithDataToSync()
+
-                        {
+                        await twitterUsersBufferBlock.SendAsync(u.ToArray(), ct);
-                            User = u,
+
-                            Followers = followers
+                        await Task.Delay(WaitFactor, ct);
-                        });
-                        
                     }
 
-                    await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
+                    var splitCount = splitUsers.Count();
+                    if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
 
+                    //// Extra wait time to fit 100.000/day limit
+                    //var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
+                    //if (extraWaitTime < 0) extraWaitTime = 0;
+                    //await Task.Delay(extraWaitTime * 1000, ct);
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(e, "Failing retrieving Twitter Users.");
                 }
-                
-                await Task.Delay(10, ct); // this is somehow necessary
             }
         }
     }
-}
+}

src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Models;
+using Microsoft.Extensions.Logging;
+
+namespace BirdsiteLive.Pipeline.Processors
+{
+    public class SaveProgressionProcessor : ISaveProgressionProcessor
+    {
+        private readonly ITwitterUserDal _twitterUserDal;
+        private readonly ILogger<SaveProgressionProcessor> _logger;
+
+        #region Ctor
+        public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionProcessor> logger)
+        {
+            _twitterUserDal = twitterUserDal;
+            _logger = logger;
+        }
+        #endregion
+
+        public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
+        {
+            try
+            {
+                if (userWithTweetsToSync.Tweets.Length == 0)
+                {
+                    _logger.LogWarning("No tweets synchronized");
+                    return;
+                }
+                if(userWithTweetsToSync.Followers.Length == 0)
+                {
+                    _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct);
+                    return;
+                }
+            
+                var userId = userWithTweetsToSync.User.Id;
+                var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
+            
+                if (followingSyncStatuses.Count == 0)
+                {
+                    _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId);
+                    return;
+                }
+
+                var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
+                var minimumSync = followingSyncStatuses.Min();
+                var now = DateTime.UtcNow;
+                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
+                throw;
+            }
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs

@@ -16,6 +16,7 @@ using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
+using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Processors
 {
@@ -27,7 +28,6 @@ namespace BirdsiteLive.Pipeline.Processors
         private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
         private readonly IRemoveFollowerAction _removeFollowerAction;
-        private List<Task> _todo = new List<Task>();
 
         #region Ctor
         public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, IFollowersDal followersDal, ILogger<SendTweetsToFollowersProcessor> logger, InstanceSettings instanceSettings, IRemoveFollowerAction removeFollowerAction)
@@ -41,41 +41,23 @@ namespace BirdsiteLive.Pipeline.Processors
         }
         #endregion
 
-        public async Task ProcessAsync(UserWithDataToSync[] usersWithTweetsToSync, CancellationToken ct)
+        public async Task<UserWithDataToSync> ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
         {
-            foreach (var userWithTweetsToSync in usersWithTweetsToSync)
+            var user = userWithTweetsToSync.User;
-            {
-                var user = userWithTweetsToSync.User;
 
-                _todo = _todo.Where(x => !x.IsCompleted).ToList();
+            // Process Shared Inbox
-                
+            var followersWtSharedInbox = userWithTweetsToSync.Followers
-                var t = Task.Run( async () => 
+                .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                {
+                .ToList();
-                    // Process Shared Inbox
+            await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
-                    var followersWtSharedInbox = userWithTweetsToSync.Followers
-                        .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                        .ToList();
-                    await ProcessFollowersWithSharedInboxAsync(userWithTweetsToSync.Tweets, followersWtSharedInbox, user);
 
-                    // Process Inbox
+            // Process Inbox
-                    var followerWtInbox = userWithTweetsToSync.Followers
+            var followerWtInbox = userWithTweetsToSync.Followers
-                        .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
+                .Where(x => string.IsNullOrWhiteSpace(x.SharedInboxRoute))
-                        .ToList();
+                .ToList();
-                    await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
+            await ProcessFollowersWithInboxAsync(userWithTweetsToSync.Tweets, followerWtInbox, user);
-                    
-                    _logger.LogInformation("Done sending " + userWithTweetsToSync.Tweets.Length + " tweets for "
-                        + userWithTweetsToSync.Followers.Length + "followers for user " + userWithTweetsToSync.User.Acct);
-                }, ct);
-                _todo.Add(t);
-
-                if (_todo.Count >= _instanceSettings.ParallelFediversePosts)
-                {
-                    await Task.WhenAny(_todo);
-                }
-                
-                
-            }
 
+            return userWithTweetsToSync;
         }
 
         private async Task ProcessFollowersWithSharedInboxAsync(ExtractedTweet[] tweets, List<Follower> followers, SyncTwitterUser user)

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs

@@ -31,6 +31,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         {
             _activityPubService = activityPubService;
             _statusService = statusService;
+            _followersDal = followersDal;
             _settings = settings;
             _logger = logger;
         }
@@ -39,32 +40,51 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         public async Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, SyncTwitterUser user)
         {
             var userId = user.Id;
-            //var fromStatusId = follower.FollowingsSyncStatus[userId];
+            var fromStatusId = follower.FollowingsSyncStatus[userId];
             var tweetsToSend = tweets
+                .Where(x => x.Id > fromStatusId)
                 .OrderBy(x => x.Id)
                 .ToList();
 
             var inbox = follower.InboxRoute;
 
-            foreach (var tweet in tweetsToSend)
+            var syncStatus = fromStatusId;
+            try
             {
-                try
+                foreach (var tweet in tweetsToSend)
                 {
-                    var activity = _statusService.GetActivity(user.Acct, tweet);
+                    try
-                    await _activityPubService.PostNewActivity(activity, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
-                }
-                catch (ArgumentException e)
-                {
-                    if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
                     {
-                        _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
+                        if (!tweet.IsReply ||
+                            tweet.IsReply && tweet.IsThread ||
+                            _settings.PublishReplies)
+                        {
+                            var note = _statusService.GetStatus(user.Acct, tweet);
+                            await _activityPubService.PostNewActivity(note, user.Acct, "Create", tweet.Id.ToString(), follower.Host, inbox);
+                        }
                     }
-                    else
+                    catch (ArgumentException e)
                     {
-                        throw;
+                        if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
+                        {
+                            _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
+                        }
+                        else
+                        {
+                            throw;
+                        }
                     }
-                }
 
+                    syncStatus = tweet.Id;
+                }
+            }
+            finally
+            {
+                if (syncStatus != fromStatusId)
+                {
+                    follower.FollowingsSyncStatus[userId] = syncStatus;
+                    await _followersDal.UpdateFollowerAsync(follower);
+                }
             }
         }
     }

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs

@@ -40,29 +40,60 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
             var userId = user.Id;
             var inbox = followersPerInstance.First().SharedInboxRoute;
 
+            var fromStatusId = followersPerInstance
+                .Max(x => x.FollowingsSyncStatus[userId]);
+
             var tweetsToSend = tweets
+                .Where(x => x.Id > fromStatusId)
                 .OrderBy(x => x.Id)
                 .ToList();
+            _logger.LogInformation("After filtering, there were " + tweetsToSend.Count() + " tweets left to send");
 
-            foreach (var tweet in tweetsToSend)
+            var syncStatus = fromStatusId;
+            try
             {
-                try
+                foreach (var tweet in tweetsToSend)
                 {
-                    var activity = _statusService.GetActivity(user.Acct, tweet);
+                    try
-                    await _activityPubService.PostNewActivity(activity, user.Acct, tweet.Id.ToString(), host, inbox);
-                }
-                catch (ArgumentException e)
-                {
-                    if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
                     {
-                        _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
+                        if (tweet.IsRetweet)
+                        {
+                            var note = _statusService.GetStatus(user.Acct, tweet);
+                            await _activityPubService.PostNewActivity(note, user.Acct, "Announce", tweet.Id.ToString(), host, inbox);
+                        }
+                        else if (!tweet.IsReply ||
+                            tweet.IsReply && tweet.IsThread ||
+                            _settings.PublishReplies)
+                        {
+                            var note = _statusService.GetStatus(user.Acct, tweet);
+                            await _activityPubService.PostNewActivity(note, user.Acct, "Create", tweet.Id.ToString(), host, inbox);
+                        }
                     }
-                    else
+                    catch (ArgumentException e)
                     {
-                        throw;
+                        if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
+                        {
+                            _logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
+                        }
+                        else
+                        {
+                            throw;
+                        }
                     }
-                }
 
+                    syncStatus = tweet.Id;
+                }
+            }
+            finally
+            {
+                if (syncStatus != fromStatusId)
+                {
+                    foreach (var f in followersPerInstance)
+                    {
+                        f.FollowingsSyncStatus[userId] = syncStatus;
+                        await _followersDal.UpdateFollowerAsync(f);
+                    }
+                }
             }
         }
     }

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -18,18 +18,22 @@ namespace BirdsiteLive.Pipeline
     public class StatusPublicationPipeline : IStatusPublicationPipeline
     {
         private readonly IRetrieveTwitterUsersProcessor _retrieveTwitterAccountsProcessor;
+        private readonly IRefreshTwitterUserStatusProcessor _refreshTwitterUserStatusProcessor;
         private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
         private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
         private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
+        private readonly ISaveProgressionProcessor _saveProgressionProcessor;
         private readonly ILogger<StatusPublicationPipeline> _logger;
 
         #region Ctor
-        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger<StatusPublicationPipeline> logger)
+        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, IRefreshTwitterUserStatusProcessor refreshTwitterUserStatusProcessor, ILogger<StatusPublicationPipeline> logger)
         {
             _retrieveTweetsProcessor = retrieveTweetsProcessor;
+            _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
             _retrieveFollowersProcessor = retrieveFollowersProcessor;
             _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
-            _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
+            _saveProgressionProcessor = saveProgressionProcessor;
+            _refreshTwitterUserStatusProcessor = refreshTwitterUserStatusProcessor;
 
             _logger = logger;
         }
@@ -37,30 +41,37 @@ namespace BirdsiteLive.Pipeline
 
         public async Task ExecuteAsync(CancellationToken ct)
         {
-            var standardBlockOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 1, MaxDegreeOfParallelism = 1, CancellationToken = ct};
             // Create blocks 
-            var twitterUserToRefreshBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions
+            var twitterUserToRefreshBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions
                 { BoundedCapacity = 1, CancellationToken = ct });
-            var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x =>  await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
+            var twitterUserToRefreshBlock = new TransformBlock<SyncTwitterUser[], UserWithDataToSync[]>(async x => await _refreshTwitterUserStatusProcessor.ProcessAsync(x, ct));
-            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 2, CancellationToken = ct });
+            var twitterUsersBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
-           // var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
+            var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
-           // var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
+            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
-            var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync[]>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);
+            var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
+            var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
+            var sendTweetsToFollowersBlock = new TransformBlock<UserWithDataToSync, UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
+            var sendTweetsToFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
+            var saveProgressionBlock = new ActionBlock<UserWithDataToSync>(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
 
             // Link pipeline
-            twitterUserToRefreshBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            twitterUserToRefreshBufferBlock.LinkTo(twitterUserToRefreshBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            twitterUserToRefreshBlock.LinkTo(twitterUsersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
-            retrieveTweetsBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            sendTweetsToFollowersBlock.LinkTo(sendTweetsToFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
+            sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true });
 
-            // Launch twitter user retriever after a little delay
+            // Launch twitter user retriever
-            // to give time for the Tweet cache to fill
-            await Task.Delay(30 * 1000, ct);
             var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUserToRefreshBufferBlock, ct);
 
             // Wait
-            await Task.WhenAny(new[] { retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion });
+            await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion });
 
-            var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : sendTweetsToFollowersBlock.Completion.Exception;
+            var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : saveProgressionBlock.Completion.Exception;
             _logger.LogCritical(ex, "An error occurred, pipeline stopped");
         }
     }

src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs

@@ -0,0 +1,49 @@
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+
+namespace BirdsiteLive.Pipeline.Tools
+{
+    public interface IMaxUsersNumberProvider
+    {
+        Task<int> GetMaxUsersNumberAsync();
+    }
+
+    public class MaxUsersNumberProvider : IMaxUsersNumberProvider
+    {
+        private readonly InstanceSettings _instanceSettings;
+        private readonly ITwitterUserDal _twitterUserDal;
+        
+        private int _totalUsersCount = -1;
+        private int _warmUpIterations;
+        private const int WarmUpMaxCapacity = 200;
+
+        #region Ctor
+        public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
+        {
+            _instanceSettings = instanceSettings;
+            _twitterUserDal = twitterUserDal;
+        }
+        #endregion
+
+        public async Task<int> GetMaxUsersNumberAsync()
+        {
+            // Init data
+            if (_totalUsersCount == -1) 
+            {
+                _totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
+                _warmUpIterations = (int)(_totalUsersCount / (float)WarmUpMaxCapacity);
+            }
+
+            // Return if warm up ended
+            if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity;
+
+            // Calculate warm up value
+            var maxUsers = _warmUpIterations > 0
+                ? WarmUpMaxCapacity
+                : _instanceSettings.MaxUsersCapacity;
+            _warmUpIterations--;
+            return maxUsers;
+        }
+    }
+}

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -1,18 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
-    <PackageReference Include="System.Threading.RateLimiting" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
-    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/BirdsiteLive.Twitter/CachedTwitterService.cs

@@ -1,6 +1,4 @@
 using System;
-using System.Text.Json;
-using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Caching.Memory;
@@ -10,8 +8,6 @@ namespace BirdsiteLive.Twitter
     public interface ICachedTwitterUserService : ITwitterUserService
     {
         void PurgeUser(string username);
-        void AddUser(TwitterUser user);
-        bool UserIsCached(string username);
     }
 
     public class CachedTwitterUserService : ICachedTwitterUserService
@@ -22,11 +18,11 @@ namespace BirdsiteLive.Twitter
         private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
             .SetSize(1)//Size amount
             //Priority on removing when reaching size limit (memory pressure)
-            .SetPriority(CacheItemPriority.Low)
+            .SetPriority(CacheItemPriority.High)
             // Keep in cache for this time, reset time if accessed.
-            .SetSlidingExpiration(TimeSpan.FromMinutes(60))
+            .SetSlidingExpiration(TimeSpan.FromHours(24))
             // Remove from cache after this time, regardless of sliding expiration
-            .SetAbsoluteExpiration(TimeSpan.FromDays(1));
+            .SetAbsoluteExpiration(TimeSpan.FromDays(7));
 
         #region Ctor
         public CachedTwitterUserService(ITwitterUserService twitterService, InstanceSettings settings)
@@ -40,19 +36,15 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
-        public bool UserIsCached(string username)
+        public TwitterUser GetUser(string username)
         {
-            return _userCache.TryGetValue(username, out _);
+            if (!_userCache.TryGetValue(username, out TwitterUser user))
-        }
-        public async Task<TwitterUser> GetUserAsync(string username)
-        {
-            if (!_userCache.TryGetValue(username, out Task<TwitterUser> user))
             {
-                user = _twitterService.GetUserAsync(username);
+                user = _twitterService.GetUser(username);
-                await _userCache.Set(username, user, _cacheEntryOptions);
+                if(user != null) _userCache.Set(username, user, _cacheEntryOptions);
             }
 
-            return await user;
+            return user;
         }
 
         public bool IsUserApiRateLimited()
@@ -60,18 +52,9 @@ namespace BirdsiteLive.Twitter
             return _twitterService.IsUserApiRateLimited();
         }
 
-        public TwitterUser Extract(JsonElement result)
-        {
-            return _twitterService.Extract(result);
-        }
         public void PurgeUser(string username)
         {
             _userCache.Remove(username);
         }
-        public void AddUser(TwitterUser user)
-        {
-
-            _userCache.Set(user.Acct, user, _cacheEntryOptions);
-        }
     }
 }

src/BirdsiteLive.Twitter/CachedTwitterTweetsService.cs

@@ -1,64 +0,0 @@
-using System;
-using System.Text.Json;
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.Twitter.Models;
-using Microsoft.Extensions.Caching.Memory;
-
-namespace BirdsiteLive.Twitter
-{
-    public interface ICachedTwitterTweetsService : ITwitterTweetsService
-    {
-        void SetTweet(long id, ExtractedTweet tweet);
-    }
-
-    public class CachedTwitterTweetsService : ICachedTwitterTweetsService
-    {
-        private readonly ITwitterTweetsService _twitterService;
-
-        private readonly MemoryCache _tweetCache;
-        private readonly MemoryCacheEntryOptions _cacheEntryOptions;
-
-        #region Ctor
-        public CachedTwitterTweetsService(ITwitterTweetsService twitterService, InstanceSettings settings)
-        {
-            _twitterService = twitterService;
-
-            _tweetCache = new MemoryCache(new MemoryCacheOptions()
-            {
-                SizeLimit = settings.TweetCacheCapacity,
-            });
-            _cacheEntryOptions = new MemoryCacheEntryOptions()
-                .SetSize(1)
-                //Priority on removing when reaching size limit (memory pressure)
-                .SetPriority(CacheItemPriority.Low)
-                // Keep in cache for this time, reset time if accessed.
-                .SetSlidingExpiration(TimeSpan.FromMinutes(60))
-                // Remove from cache after this time, regardless of sliding expiration
-                .SetAbsoluteExpiration(TimeSpan.FromDays(1));
-        }
-        #endregion
-
-        public async Task<ExtractedTweet[]> GetTimelineAsync(string username, long id)
-        {
-            var res = await _twitterService.GetTimelineAsync(username, id);
-            return res;
-        }
-        public async Task<ExtractedTweet> GetTweetAsync(long id)
-        {
-            if (!_tweetCache.TryGetValue(id, out Task<ExtractedTweet> tweet))
-            {
-                tweet = _twitterService.GetTweetAsync(id);
-                await _tweetCache.Set(id, tweet, _cacheEntryOptions);
-            }
-
-            return await tweet;
-        }
-
-        public void SetTweet(long id, ExtractedTweet tweet)
-        {
-
-            _tweetCache.Set(id, tweet, _cacheEntryOptions);
-        }
-    }
-}

src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs

@@ -4,6 +4,5 @@
     {
         public string MediaType { get; set; }
         public string Url { get; set; }
-        public string AltText { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs

@@ -17,6 +17,5 @@ namespace BirdsiteLive.Twitter.Models
         public string RetweetUrl { get; set; }
         public long RetweetId { get; set; }
         public TwitterUser OriginalAuthor { get; set; }
-        public TwitterUser Author { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -1,166 +1,77 @@
 using System;
 using System.Threading;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
 using System.Net.Http;
-using System.Net;
 using System.Net.Http.Headers;
-using System.Text;
 using System.Text.Json;
-using System.Threading.RateLimiting;
 
 namespace BirdsiteLive.Twitter.Tools
 {
     public interface ITwitterAuthenticationInitializer
     {
-        Task<HttpClient> MakeHttpClient();
+        String Token { get; }
-        HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken);
+        Task EnsureAuthenticationIsInitialized();
-        Task RefreshClient(HttpRequestMessage client);
     }
 
     public class TwitterAuthenticationInitializer : ITwitterAuthenticationInitializer
     {
+        private readonly TwitterSettings _settings;
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
-        private readonly IHttpClientFactory _httpClientFactory;
+        private readonly HttpClient _httpClient = new HttpClient();
-        private ConcurrentDictionary<String, String> _token2 = new ConcurrentDictionary<string, string>();
+        private String _token;
-        static Random rnd = new Random();
+        public String Token { 
-        private RateLimiter _rateLimiter;
+            get { return _token; }
-        private const int _targetClients = 3;
-        private InstanceSettings _instanceSettings;
-        private readonly (string, string)[] _apiKeys = new[]
-        {
-            ("IQKbtAYlXLripLGPWd0HUA", "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU"), // iPhone
-            ("3nVuSoBZnx6U4vzUxf5w", "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"), // Android
-            ("CjulERsDeqhhjSme66ECg", "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck"), // iPad
-            ("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
-        };
-        
-        private readonly string[] _bTokens =  new[] 
-                {
-            // developer.twitter.com
-                    "AAAAAAAAAAAAAAAAAAAAACHguwAAAAAAaSlT0G31NDEyg%2BSnBN5JuyKjMCU%3Dlhg0gv0nE7KKyiJNEAojQbn8Y3wJm1xidDK7VnKGBP4ByJwHPb",
-            // tweetdeck new
-                    "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF",
-            // ipad -- TimelineSearch returns data in a different format, making nitter return empty results. on the other hand, it has high rate limits. build separate token pools per endpoint?
-                    "AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR",
-        };
-        public String BearerToken { 
-            get
-            {
-                return _instanceSettings.TwitterBearerToken;
-            }
         }
 
         #region Ctor
-        public TwitterAuthenticationInitializer(IHttpClientFactory httpClientFactory, InstanceSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
+        public TwitterAuthenticationInitializer(TwitterSettings settings, ILogger<TwitterAuthenticationInitializer> logger)
         {
+            _settings = settings;
             _logger = logger;
-            _instanceSettings = settings;
-            _httpClientFactory = httpClientFactory;
-
-            var concuOpt = new ConcurrencyLimiterOptions();
-            concuOpt.PermitLimit = 1;
-            _rateLimiter = new ConcurrencyLimiter(concuOpt);
         }
         #endregion
 
-        private async Task<string> GenerateBearerToken()
+        public async Task EnsureAuthenticationIsInitialized()
         {
-            var httpClient = _httpClientFactory.CreateClient();
+            if (_initialized) return;
-            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
+           
+            await InitTwitterCredentials();
+        }
+
+        private async Task InitTwitterCredentials()
+        {
+            for (;;)
             {
-                int r1 = rnd.Next(_bTokens.Length);
+                try
-                return _bTokens[r1];
+                {
-                
-                int r = rnd.Next(_apiKeys.Length);
-                var (login, password) = _apiKeys[r];
-                var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
-                request.Headers.Authorization = authValue;
 
-                var httpResponse = await httpClient.SendAsync(request);
+                    using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token"))
+                    {
+                        var base64authorization = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes(_settings.ConsumerKey + ":" + _settings.ConsumerSecret));
+                        request.Headers.TryAddWithoutValidation("Authorization", $"Basic {base64authorization}"); 
 
-                var c = await httpResponse.Content.ReadAsStringAsync();
+                        request.Content = new StringContent("grant_type=client_credentials");
-                httpResponse.EnsureSuccessStatusCode();
+                        request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/x-www-form-urlencoded"); 
-                var doc = JsonDocument.Parse(c);
+
-                var token = doc.RootElement.GetProperty("access_token").GetString();
+                        var httpResponse = await _httpClient.SendAsync(request);
-                return token;
+
+                        var c = await httpResponse.Content.ReadAsStringAsync();
+                        httpResponse.EnsureSuccessStatusCode();
+                        var doc = JsonDocument.Parse(c);
+                        _token = doc.RootElement.GetProperty("access_token").GetString();
+                    }
+                    _initialized = true;
+                    return;
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(e, "Twitter Authentication Failed");
+                    await Task.Delay(3600*1000);
+                }
             }
-            
-        }
-
-
-        public async Task RefreshClient(HttpRequestMessage req)
-        {
-            string token = req.Headers.GetValues("x-guest-token").First();
-
-            _token2.TryRemove(token, out _);
-
-            await RefreshCred();
-            await Task.Delay(1000);
-            await RefreshCred();
-        }
-
-        private async Task RefreshCred()
-        {
-            (string bearer, string guest) = await GetCred();
-            _token2.TryAdd(guest, bearer);
-        }
-
-        private async Task<(string, string)> GetCred()
-        {
-            string token;
-            var httpClient = _httpClientFactory.CreateClient();
-            string bearer = await GenerateBearerToken();
-            using RateLimitLease lease = await _rateLimiter.AcquireAsync(permitCount: 1);
-            using var request = new HttpRequestMessage(new HttpMethod("POST"),
-                "https://api.twitter.com/1.1/guest/activate.json");
-            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
-            //request.Headers.Add("User-Agent",
-            //    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36");
-
-            HttpResponseMessage httpResponse;
-            do
-            {
-                httpResponse = await httpClient.SendAsync(request);
-
-                var c = await httpResponse.Content.ReadAsStringAsync();
-                if (httpResponse.StatusCode == HttpStatusCode.TooManyRequests)
-                    await Task.Delay(1000);
-                httpResponse.EnsureSuccessStatusCode();
-                var doc = JsonDocument.Parse(c);
-                token = doc.RootElement.GetProperty("guest_token").GetString();
-
-            } while (httpResponse.StatusCode != HttpStatusCode.OK);
-
-            return (bearer, token);
-
-        }
-
-        public async Task<HttpClient> MakeHttpClient()
-        {
-            if (_token2.Count < _targetClients)
-                await RefreshCred();
-            return _httpClientFactory.CreateClient();
-        }
-        public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
-        {
-            var request = new HttpRequestMessage(m, endpoint);
-            (string token, string bearer) = _token2.MaxBy(x => rnd.Next());
-            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
-            request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
-            request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
-            //request.Headers.Add("User-Agent",
-            //    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36");
-            if (addToken)
-                request.Headers.TryAddWithoutValidation("x-guest-token", token);
-            //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
-            //request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
-            return request;
         }
     }
 }

src/BirdsiteLive.Twitter/TwitterTweetsService.cs

@@ -2,10 +2,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.IO;
-using System.Net;
 using System.Net.Http;
 using System.Text.Json;
-using System.Text.Json.Nodes;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Statistics.Domain;
@@ -13,169 +11,116 @@ using BirdsiteLive.Twitter.Models;
 using BirdsiteLive.Twitter.Tools;
 using Microsoft.Extensions.Logging;
 using System.Text.RegularExpressions;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
 
 namespace BirdsiteLive.Twitter
 {
     public interface ITwitterTweetsService
     {
-        Task<ExtractedTweet> GetTweetAsync(long statusId);
+        ExtractedTweet GetTweet(long statusId);
-        Task<ExtractedTweet[]> GetTimelineAsync(string username, long fromTweetId = -1);
+        ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1);
     }
 
     public class TwitterTweetsService : ITwitterTweetsService
     {
         private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
         private readonly ITwitterStatisticsHandler _statisticsHandler;
-        private readonly ICachedTwitterUserService _twitterUserService;
+        private readonly ITwitterUserService _twitterUserService;
-        private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<TwitterTweetsService> _logger;
-        private readonly InstanceSettings _instanceSettings;
+        private HttpClient _httpClient = new HttpClient();
-        private static string gqlFeatures = """
-        { 
-          "android_graphql_skip_api_media_color_palette": false,
-          "blue_business_profile_image_shape_enabled": false,
-          "creator_subscriptions_subscription_count_enabled": false,
-          "creator_subscriptions_tweet_preview_api_enabled": true,
-          "freedom_of_speech_not_reach_fetch_enabled": false,
-          "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
-          "hidden_profile_likes_enabled": false,
-          "highlights_tweets_tab_ui_enabled": false,
-          "interactive_text_enabled": false,
-          "longform_notetweets_consumption_enabled": true,
-          "longform_notetweets_inline_media_enabled": false,
-          "longform_notetweets_richtext_consumption_enabled": true,
-          "longform_notetweets_rich_text_read_enabled": false,
-          "responsive_web_edit_tweet_api_enabled": false,
-          "responsive_web_enhance_cards_enabled": false,
-          "responsive_web_graphql_exclude_directive_enabled": true,
-          "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
-          "responsive_web_graphql_timeline_navigation_enabled": false,
-          "responsive_web_media_download_video_enabled": false,
-          "responsive_web_text_conversations_enabled": false,
-          "responsive_web_twitter_article_tweet_consumption_enabled": false,
-          "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-          "rweb_lists_timeline_redesign_enabled": true,
-          "spaces_2022_h2_clipping": true,
-          "spaces_2022_h2_spaces_communities": true,
-          "standardized_nudges_misinfo": false,
-          "subscriptions_verification_info_enabled": true,
-          "subscriptions_verification_info_reason_enabled": true,
-          "subscriptions_verification_info_verified_since_enabled": true,
-          "super_follow_badge_privacy_enabled": false,
-          "super_follow_exclusive_tweet_notifications_enabled": false,
-          "super_follow_tweet_api_enabled": false,
-          "super_follow_user_api_enabled": false,
-          "tweet_awards_web_tipping_enabled": false,
-          "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
-          "tweetypie_unmention_optimization_enabled": false,
-          "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-          "verified_phone_label_enabled": false,
-          "vibe_api_enabled": false,
-          "view_counts_everywhere_api_enabled": false
-        }
-        """.Replace(" ", "").Replace("\n", "");
 
         #region Ctor
-        public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<TwitterTweetsService> logger)
+        public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, ILogger<TwitterTweetsService> logger)
         {
             _twitterAuthenticationInitializer = twitterAuthenticationInitializer;
             _statisticsHandler = statisticsHandler;
             _twitterUserService = twitterUserService;
-            _twitterUserDal = twitterUserDal;
-            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
 
 
+        public ExtractedTweet GetTweet(long statusId)
+        {
+            return GetTweetAsync(statusId).Result;
+        }
         public async Task<ExtractedTweet> GetTweetAsync(long statusId)
         {
-
-            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
-
-
-            // https://platform.twitter.com/embed/Tweet.html?id=1633788842770825216
-            string reqURL =
-                "https://api.twitter.com/graphql/83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2?variables={%22focalTweetId%22%3A%22"
-                + statusId +
-                "%22,%22count%22:20,%22includeHasBirdwatchNotes%22:false}&features="+ gqlFeatures;
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
+                await _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
                 JsonDocument tweet;
-                var httpResponse = await client.SendAsync(request);
+                var reqURL = "https://api.twitter.com/2/tweets/"  + statusId
-                if (httpResponse.StatusCode == HttpStatusCode.Unauthorized)
+                     + "?expansions=author_id,referenced_tweets.id,attachments.media_keys,entities.mentions.username,referenced_tweets.id.author_id&tweet.fields=id,created_at,text,author_id,in_reply_to_user_id,referenced_tweets,attachments,withheld,geo,entities,public_metrics,possibly_sensitive,source,lang,context_annotations,conversation_id,reply_settings&user.fields=id,created_at,name,username,protected,verified,withheld,profile_image_url,location,url,description,entities,pinned_tweet_id,public_metrics&media.fields=media_key,duration_ms,height,preview_image_url,type,url,width,public_metrics,alt_text,variants";
-                {
+                using (var request = new HttpRequestMessage(new HttpMethod("GET"), reqURL))
-                    _logger.LogError("Error retrieving tweet {statusId}; refreshing client", statusId);
+    {
-                    await _twitterAuthenticationInitializer.RefreshClient(request);
+                    request.Headers.TryAddWithoutValidation("Authorization", "Bearer " + _twitterAuthenticationInitializer.Token); 
+
+                    var httpResponse = await _httpClient.SendAsync(request);
+                    httpResponse.EnsureSuccessStatusCode();
+                    var c = await httpResponse.Content.ReadAsStringAsync();
+                    tweet = JsonDocument.Parse(c);
                 }
-                httpResponse.EnsureSuccessStatusCode();
-                var c = await httpResponse.Content.ReadAsStringAsync();
-                tweet = JsonDocument.Parse(c);
 
+                _statisticsHandler.CalledTweetApi();
+                if (tweet == null) return null; //TODO: test this
 
-                var timeline = tweet.RootElement.GetProperty("data").GetProperty("timeline_response")
+                JsonElement mediaExpension = default;
-                    .GetProperty("instructions").EnumerateArray().First().GetProperty("entries").EnumerateArray();
+                try 
+                {
+                    tweet.RootElement.GetProperty("includes").TryGetProperty("media", out mediaExpension);
+                } 
+                catch (Exception)
+                { }
 
-                var tweetInDoc = timeline.Where(x => x.GetProperty("entryId").GetString() == "tweet-" + statusId)
+                //return tweet.RootElement.GetProperty("data").EnumerateArray().Select<JsonElement, ExtractedTweet>(x => Extract(x, mediaExpension)).ToArray().First();
-                    .ToArray().First();
+                return Extract( tweet.RootElement.GetProperty("data"), mediaExpension);
-                return await Extract( tweetInDoc );
             }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving tweet {TweetId}", statusId);
-                await _twitterAuthenticationInitializer.RefreshClient(request);
                 return null;
             }
         }
 
-        public async Task<ExtractedTweet[]> GetTimelineAsync(string username, long fromTweetId = -1)
+        public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
         {
+            return GetTimelineAsync(username, nberTweets, fromTweetId).Result;
+        }
+        public async Task<ExtractedTweet[]> GetTimelineAsync(string username, int nberTweets, long fromTweetId = -1)
+        {
+            if (nberTweets < 5)
+                nberTweets = 5;
 
-            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
+            if (nberTweets > 100)
+                nberTweets = 100;
 
-            long userId;
+            await _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
-            SyncTwitterUser user = await _twitterUserDal.GetTwitterUserAsync(username);
-            if (user.TwitterUserId == default) 
-            {
-                var user2 = await _twitterUserService.GetUserAsync(username);
-                userId = user2.Id;
-                await _twitterUserDal.UpdateTwitterUserIdAsync(username, user2.Id);
-            }
-            else 
-            {
-                userId = user.TwitterUserId;
-            }
 
+            var user = _twitterUserService.GetUser(username);
+            if (user == null || user.Protected) return new ExtractedTweet[0];
 
-            var reqURL =
+            var reqURL = "https://api.twitter.com/2/users/" 
-                "https://api.twitter.com/graphql/8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2?variables=%7B%22rest_id%22%3A%22" +
+                 + user.Id + 
-                userId +
+                 "/tweets?expansions=in_reply_to_user_id,attachments.media_keys,entities.mentions.username,referenced_tweets.id.author_id"
-                "%22,%22count%22%3A40,%22includeHasBirdwatchNotes%22%3Atrue}&features=" +
+                 + "&tweet.fields=id,created_at"
-                gqlFeatures;
+                 + "&media.fields=media_key,duration_ms,height,preview_image_url,type,url,width,public_metrics,alt_text,variants"
-            //reqURL =
+                 + "&max_results=" + nberTweets
-            //    """https://twitter.com/i/api/graphql/rIIwMe1ObkGh_ByBtTCtRQ/UserTweets?variables={"userId":"44196397","count":20,"includePromotedContent":true,"withQuickPromoteEligibilityTweetFields":true,"withVoice":true,"withV2Timeline":true}&features={"rweb_lists_timeline_redesign_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":false,"tweet_awards_web_tipping_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_media_download_video_enabled":false,"responsive_web_enhance_cards_enabled":false}&fieldToggles={"withArticleRichContentState":false}""";
+                 + "" ; // ?since_id=2324234234
-            //reqURL = reqURL.Replace("44196397", userId.ToString());
+            JsonDocument tweets;
-            JsonDocument results;
-            List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
+                using (var request = new HttpRequestMessage(new HttpMethod("GET"), reqURL))
+    {
+                    request.Headers.TryAddWithoutValidation("Authorization", "Bearer " + _twitterAuthenticationInitializer.Token); 
 
-                var httpResponse = await client.SendAsync(request);
+                    var httpResponse = await _httpClient.SendAsync(request);
-                var c = await httpResponse.Content.ReadAsStringAsync();
+                    httpResponse.EnsureSuccessStatusCode();
-                if (httpResponse.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden)
+                    var c = await httpResponse.Content.ReadAsStringAsync();
-                {
+                    tweets = JsonDocument.Parse(c);
-                    _logger.LogError("Error retrieving timeline of {Username}; refreshing client", username);
-                    await _twitterAuthenticationInitializer.RefreshClient(request);
-                    return null;
                 }
-                httpResponse.EnsureSuccessStatusCode();
-                results = JsonDocument.Parse(c);
 
                 _statisticsHandler.CalledTweetApi();
+                if (tweets == null) return null; //TODO: test this
             }
             catch (Exception e)
             {
@@ -183,237 +128,111 @@ namespace BirdsiteLive.Twitter
                 return null;
             }
 
-            var timeline = results.RootElement.GetProperty("data").GetProperty("user_result").GetProperty("result")
+            JsonElement mediaExpension = default;
-                .GetProperty("timeline_response").GetProperty("timeline").GetProperty("instructions").EnumerateArray();
+            try 
-
-            foreach (JsonElement timelineElement in timeline) 
             {
-                if (timelineElement.GetProperty("__typename").GetString() != "TimelineAddEntries")
+                tweets.RootElement.GetProperty("includes").TryGetProperty("media", out mediaExpension);
-                    continue;
+            } 
+            catch (Exception)
+            { }
 
-                
-                foreach (JsonElement tweet in timelineElement.GetProperty("entries").EnumerateArray())
-                {
-                    if (tweet.GetProperty("content").GetProperty("__typename").GetString() != "TimelineTimelineItem")
-                        continue;
-                    
 
-                    try 
+            return tweets.RootElement.GetProperty("data").EnumerateArray().Select<JsonElement, ExtractedTweet>(x => Extract(x, mediaExpension)).ToArray();
-                    {   
-                        var extractedTweet = await Extract(tweet);
-
-                        if (extractedTweet.Id == fromTweetId)
-                            break;
-
-                        extractedTweets.Add(extractedTweet);
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogError("Tried getting timeline from user " + username + ", but got error: \n" +
-                                         e.Message + e.StackTrace + e.Source);
-
-                    }
-
-                }
-            }
-
-            return extractedTweets.ToArray();
         }
 
-        private async Task<ExtractedTweet> Extract(JsonElement tweet)
+        private ExtractedTweet Extract(JsonElement tweet, JsonElement media)
         {
-
+            var id = Int64.Parse(tweet.GetProperty("id").GetString());
-            JsonElement retweet;
+            bool IsRetweet = false;
-            TwitterUser OriginalAuthor;
+            bool IsReply = false;
-            TwitterUser author = null;
+            long? replyId = null;
-            JsonElement inReplyToPostIdElement;
+            JsonElement replyAccount;
-            JsonElement inReplyToUserElement;
+            string? replyAccountString = null;
-            string inReplyToUser = null;
+            JsonElement referenced_tweets;
-            long? inReplyToPostId = null;
+            if(tweet.TryGetProperty("in_reply_to_user_id", out replyAccount))
-            long retweetId = default;
-
-            string userName = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("core").GetProperty("user_result")
-                    .GetProperty("result").GetProperty("legacy").GetProperty("screen_name").GetString();
-
-            JsonElement userDoc = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("core")
-                    .GetProperty("user_result").GetProperty("result");
-
-            author = _twitterUserService.Extract(userDoc); 
-            
-            bool isReply = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .TryGetProperty("in_reply_to_status_id_str", out inReplyToPostIdElement);
-            tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .TryGetProperty("in_reply_to_screen_name", out inReplyToUserElement);
-            if (isReply) 
             {
-                inReplyToPostId = Int64.Parse(inReplyToPostIdElement.GetString());
+                replyAccountString = replyAccount.GetString();
-                inReplyToUser = inReplyToUserElement.GetString();
+
             }
-            bool isRetweet = tweet.GetProperty("content").GetProperty("content")
+            if(tweet.TryGetProperty("referenced_tweets", out referenced_tweets))
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .TryGetProperty("retweeted_status_result", out retweet);
-            string MessageContent;
-            if (!isRetweet)
             {
-                MessageContent = tweet.GetProperty("content").GetProperty("content")
+                var first = referenced_tweets.EnumerateArray().ToList()[0];
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+                if (first.GetProperty("type").GetString() == "retweeted")
-                    .GetProperty("full_text").GetString();
-                bool isNote = tweet.GetProperty("content").GetProperty("content")
-                        .GetProperty("tweetResult").GetProperty("result")
-                        .TryGetProperty("note_tweet", out var note);
-                if (isNote)
                 {
-                    MessageContent = note.GetProperty("note_tweet_results").GetProperty("result")
+                    IsRetweet = true;
-                        .GetProperty("text").GetString();
+                    var regex = new Regex("RT @([A-Za-z0-9_]+):");
+                    var match = regex.Match(tweet.GetProperty("text").GetString());
+                    var originalAuthor = _twitterUserService.GetUser(match.Groups[1].Value);
+                    var statusId = Int64.Parse(first.GetProperty("id").GetString());
+                    var extracted = GetTweet(statusId);
+                    extracted.RetweetId = id;
+                    extracted.IsRetweet = true;
+                    extracted.OriginalAuthor = originalAuthor;
+                    return extracted;
+
                 }
-                OriginalAuthor = null;
+                if (first.GetProperty("type").GetString() == "replied_to")
-                
-            }
-            else 
-            {
-                MessageContent = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("retweeted_status_result").GetProperty("result")
-                    .GetProperty("legacy").GetProperty("full_text").GetString();
-                bool isNote = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("retweeted_status_result").GetProperty("result")
-                    .TryGetProperty("note_tweet", out var note);
-                if (isNote)
                 {
-                    MessageContent = note.GetProperty("note_tweet_results").GetProperty("result")
+                    IsReply = true;
-                        .GetProperty("text").GetString();
+                    replyId = Int64.Parse(first.GetProperty("id").GetString());
                 }
-                string OriginalAuthorUsername = tweet.GetProperty("content").GetProperty("content")
+                if (first.GetProperty("type").GetString() == "quoted")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("retweeted_status_result").GetProperty("result")
-                    .GetProperty("core").GetProperty("user_result").GetProperty("result")
-                    .GetProperty("legacy").GetProperty("screen_name").GetString();
-                JsonElement OriginalAuthorDoc = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("retweeted_status_result").GetProperty("result")
-                    .GetProperty("core").GetProperty("user_result").GetProperty("result");
-                OriginalAuthor = _twitterUserService.Extract(OriginalAuthorDoc); 
-                //OriginalAuthor = await _twitterUserService.GetUserAsync(OriginalAuthorUsername);
-                retweetId = Int64.Parse(tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("retweeted_status_result").GetProperty("result")
-                    .GetProperty("rest_id").GetString());
-            }
-
-            string creationTime = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("created_at").GetString().Replace(" +0000", "");
-
-            JsonElement extendedEntities;
-            bool hasMedia = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .TryGetProperty("extended_entities", out extendedEntities);
-
-            JsonElement.ArrayEnumerator urls = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("entities").GetProperty("urls").EnumerateArray();
-            foreach (JsonElement url in urls)
-            {
-                string tco = url.GetProperty("url").GetString();
-                string goodUrl = url.GetProperty("expanded_url").GetString();
-                MessageContent = MessageContent.Replace(tco, goodUrl);
-            }
-            
-            List<ExtractedMedia> Media = new List<ExtractedMedia>();
-            if (hasMedia) 
-            {
-                foreach (JsonElement media in extendedEntities.GetProperty("media").EnumerateArray())
                 {
-                    var type = media.GetProperty("type").GetString();
+                    IsReply = true;
-                    string url = "";
+                    replyId = Int64.Parse(first.GetProperty("id").GetString());
-                    string altText = null;
+                }
-                    if (media.TryGetProperty("video_info", out _))
+            }
+
+            var extractedMedia = Array.Empty<ExtractedMedia>();
+            JsonElement attachments;
+            try 
+            {
+                if (tweet.TryGetProperty("attachments", out attachments))
+                {
+                    foreach (JsonElement m in attachments.GetProperty("media_keys").EnumerateArray())
                     {
-                        var bitrate = -1;
+                        var mediaInfo = media.EnumerateArray().Where(x => x.GetProperty("media_key").GetString() == m.GetString()).First();
-                        foreach (JsonElement v in media.GetProperty("video_info").GetProperty("variants").EnumerateArray())
+                        var mediaType = mediaInfo.GetProperty("type").GetString();
+                        if (mediaType != "photo")
                         {
-                            if (v.GetProperty("content_type").GetString() !=  "video/mp4")
+                            continue;
-                                continue;
-                            int vBitrate = v.GetProperty("bitrate").GetInt32();
-                            if (vBitrate > bitrate)
-                            {
-                                bitrate = vBitrate;
-                                url = v.GetProperty("url").GetString();
-                            }
                         }
-                    }
+                        var url = mediaInfo.GetProperty("url").GetString();
-                    else 
+                        extractedMedia.Append(
-                    {
+                            new ExtractedMedia 
-                        url = media.GetProperty("media_url_https").GetString();
+                            {
-                    }
+                                Url = url,
+                                MediaType = GetMediaType(mediaType, url),
+                            }
+                        );
 
-                    if (media.TryGetProperty("ext_alt_text", out JsonElement altNode))
-                    {
-                        altText = altNode.GetString();
                     }
-                    var m = new ExtractedMedia
-                    {
-                        MediaType = GetMediaType(type, url),
-                        Url = url,
-                        AltText = altText
-                    };
-                    Media.Add(m);
-
-                    MessageContent = MessageContent.Replace(media.GetProperty("url").GetString(), "");
                 }
+
             }
-
+            catch (Exception e)
-            bool isQuoteTweet = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("is_quote_status").GetBoolean();
-
-            if (isQuoteTweet) 
             {
+                _logger.LogError("Tried getting media from tweet " + id + ", but got error: \n" + e.Message + e.StackTrace + e.Source);
 
-                string quoteTweetId = tweet.GetProperty("content").GetProperty("content")
-                        .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                        .GetProperty("quoted_status_id_str").GetString();
-                string quoteTweetAcct = tweet.GetProperty("content").GetProperty("content")
-                        .GetProperty("tweetResult").GetProperty("result")
-                        .GetProperty("quoted_status_result").GetProperty("result")
-                        .GetProperty("core").GetProperty("user_result").GetProperty("result")
-                        .GetProperty("legacy").GetProperty("screen_name").GetString();
-                //Uri test = new Uri(quoteTweetLink);
-                //string quoteTweetAcct = test.Segments[1].Replace("/", "");
-                //string quoteTweetId = test.Segments[3];
-                
-                string quoteTweetLink = $"https://{_instanceSettings.Domain}/@{quoteTweetAcct}/{quoteTweetId}";
-
-                //MessageContent.Replace($"https://twitter.com/i/web/status/{}", "");
-                MessageContent = MessageContent.Replace($"https://twitter.com/{quoteTweetAcct}/status/{quoteTweetId}", "");
-                
-                MessageContent = MessageContent + "\n\n" + quoteTweetLink;
             }
-            
+
+
             var extractedTweet = new ExtractedTweet
             {
-                Id = Int64.Parse(tweet.GetProperty("entryId").GetString().Replace("tweet-", "")),
+                Id = id,
-                InReplyToStatusId = inReplyToPostId,
+                InReplyToStatusId = replyId,
-                InReplyToAccount = inReplyToUser,
+                InReplyToAccount = replyAccountString,
-                MessageContent = MessageContent.Trim(),
+                MessageContent = tweet.GetProperty("text").GetString(),
-                CreatedAt = DateTime.ParseExact(creationTime, "ddd MMM dd HH:mm:ss yyyy", System.Globalization.CultureInfo.InvariantCulture),
+                CreatedAt = tweet.GetProperty("created_at").GetDateTime(),
-                IsReply = isReply,
+                IsReply = IsReply,
-                IsThread = userName == inReplyToUser,
+                IsThread = false,
-                IsRetweet = isRetweet,
+                IsRetweet = IsRetweet,
-                Media = Media.Count() == 0 ? null : Media.ToArray(),
+                Media = extractedMedia,
                 RetweetUrl = "https://t.co/123",
-                RetweetId = retweetId,
+                OriginalAuthor = null,
-                OriginalAuthor = OriginalAuthor,
-                Author = author,
             };
-       
+
             return extractedTweet;
-         
         }
         private string GetMediaType(string mediaType, string mediaUrl)
         {

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Linq;
-using System.Net;
 using System.Net.Http;
 using System.Text.Json;
 using System.Threading.Tasks;
@@ -14,8 +13,7 @@ namespace BirdsiteLive.Twitter
 {
     public interface ITwitterUserService
     {
-        Task<TwitterUser> GetUserAsync(string username);
+        TwitterUser GetUser(string username);
-        TwitterUser Extract (JsonElement result);
         bool IsUserApiRateLimited();
     }
 
@@ -24,54 +22,7 @@ namespace BirdsiteLive.Twitter
         private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
         private readonly ITwitterStatisticsHandler _statisticsHandler;
         private readonly ILogger<TwitterUserService> _logger;
-
+        private HttpClient _httpClient = new HttpClient();
-        private readonly string endpoint =
-            "https://api.twitter.com/graphql/pVrmNaXcxPjisIvKtLDMEA/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=" + gqlFeatures;
-
-        private static string gqlFeatures = """
-        { 
-          "android_graphql_skip_api_media_color_palette": false,
-          "blue_business_profile_image_shape_enabled": false,
-          "creator_subscriptions_subscription_count_enabled": false,
-          "creator_subscriptions_tweet_preview_api_enabled": true,
-          "freedom_of_speech_not_reach_fetch_enabled": false,
-          "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
-          "hidden_profile_likes_enabled": false,
-          "highlights_tweets_tab_ui_enabled": false,
-          "interactive_text_enabled": false,
-          "longform_notetweets_consumption_enabled": true,
-          "longform_notetweets_inline_media_enabled": false,
-          "longform_notetweets_richtext_consumption_enabled": true,
-          "longform_notetweets_rich_text_read_enabled": false,
-          "responsive_web_edit_tweet_api_enabled": false,
-          "responsive_web_enhance_cards_enabled": false,
-          "responsive_web_graphql_exclude_directive_enabled": true,
-          "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
-          "responsive_web_graphql_timeline_navigation_enabled": false,
-          "responsive_web_media_download_video_enabled": false,
-          "responsive_web_text_conversations_enabled": false,
-          "responsive_web_twitter_article_tweet_consumption_enabled": false,
-          "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-          "rweb_lists_timeline_redesign_enabled": true,
-          "spaces_2022_h2_clipping": true,
-          "spaces_2022_h2_spaces_communities": true,
-          "standardized_nudges_misinfo": false,
-          "subscriptions_verification_info_enabled": true,
-          "subscriptions_verification_info_reason_enabled": true,
-          "subscriptions_verification_info_verified_since_enabled": true,
-          "super_follow_badge_privacy_enabled": false,
-          "super_follow_exclusive_tweet_notifications_enabled": false,
-          "super_follow_tweet_api_enabled": false,
-          "super_follow_user_api_enabled": false,
-          "tweet_awards_web_tipping_enabled": false,
-          "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
-          "tweetypie_unmention_optimization_enabled": false,
-          "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-          "verified_phone_label_enabled": false,
-          "vibe_api_enabled": false,
-          "view_counts_everywhere_api_enabled": false
-        }
-        """.Replace(" ", "").Replace("\n", "");
 
         #region Ctor
         public TwitterUserService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ILogger<TwitterUserService> logger)
@@ -82,32 +33,35 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
+        public TwitterUser GetUser(string username)
+        {
+            return GetUserAsync(username).Result;
+        }
         public async Task<TwitterUser> GetUserAsync(string username)
         {
+            //Check if API is saturated 
+            if (IsUserApiRateLimited()) throw new RateLimitExceededException();
+
+            //Proceed to account retrieval
+            await _twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
 
             JsonDocument res;
-            var client = await _twitterAuthenticationInitializer.MakeHttpClient();
-            using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), endpoint.Replace("elonmusk", username), true);
             try
             {
+                using (var request = new HttpRequestMessage(new HttpMethod("GET"), "https://api.twitter.com/2/users/by/username/"+ username + "?user.fields=name,username,protected,profile_image_url,url,description"))
+    {
+                    request.Headers.TryAddWithoutValidation("Authorization", "Bearer " + _twitterAuthenticationInitializer.Token); 
 
-                var httpResponse = await client.SendAsync(request);
+                    var httpResponse = await _httpClient.SendAsync(request);
-                if (httpResponse.StatusCode == HttpStatusCode.Unauthorized)
+                    httpResponse.EnsureSuccessStatusCode();
-                {
+
-                    _logger.LogError("Error retrieving user {Username}, Refreshing client", username);
+                    var c = await httpResponse.Content.ReadAsStringAsync();
-                    await _twitterAuthenticationInitializer.RefreshClient(request);
+                    res = JsonDocument.Parse(c);
-                    return null;
                 }
-                httpResponse.EnsureSuccessStatusCode();
-
-                var c = await httpResponse.Content.ReadAsStringAsync();
-                res = JsonDocument.Parse(c);
-                var result = res.RootElement.GetProperty("data").GetProperty("user").GetProperty("result");
-                return Extract(result);
             }
-            catch (System.Collections.Generic.KeyNotFoundException)
+            catch (HttpRequestException e)
             {
-                throw new UserNotFoundException();
+                throw;
                 //if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("User has been suspended".ToLowerInvariant())))
                 //{
                 //    throw new UserHasBeenSuspendedException();
@@ -116,6 +70,10 @@ namespace BirdsiteLive.Twitter
                 //{
                 //    throw new UserNotFoundException();
                 //}
+                //else if (e.TwitterExceptionInfos.Any(x => x.Message.ToLowerInvariant().Contains("Rate limit exceeded".ToLowerInvariant())))
+                //{
+                //    throw new RateLimitExceededException();
+                //}
                 //else
                 //{
                 //    throw;
@@ -136,34 +94,46 @@ namespace BirdsiteLive.Twitter
             //foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
             //    description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
 
-        }
-
-        public TwitterUser Extract(JsonElement result)
-        {
-            string profileBannerURL = null;
-            JsonElement profileBannerURLObject;
-            if (result.GetProperty("legacy").TryGetProperty("profile_banner_url", out profileBannerURLObject))
-            {
-                profileBannerURL = profileBannerURLObject.GetString();
-            }
-
             return new TwitterUser
             {
-                Id = long.Parse(result.GetProperty("rest_id").GetString()),
+                Id = long.Parse(res.RootElement.GetProperty("data").GetProperty("id").GetString()),
-                Acct = result.GetProperty("legacy").GetProperty("screen_name").GetString(), 
+                Acct = res.RootElement.GetProperty("data").GetProperty("username").GetString(),
-                Name =  result.GetProperty("legacy").GetProperty("name").GetString(), //res.RootElement.GetProperty("data").GetProperty("name").GetString(),
+                Name = res.RootElement.GetProperty("data").GetProperty("name").GetString(),
-                Description =  "", //res.RootElement.GetProperty("data").GetProperty("description").GetString(),
+                Description = res.RootElement.GetProperty("data").GetProperty("description").GetString(),
-                Url =  "", //res.RootElement.GetProperty("data").GetProperty("url").GetString(),
+                Url = res.RootElement.GetProperty("data").GetProperty("url").GetString(),
-                ProfileImageUrl =  result.GetProperty("legacy").GetProperty("profile_image_url_https").GetString().Replace("normal", "400x400"), 
+                ProfileImageUrl = res.RootElement.GetProperty("data").GetProperty("profile_image_url").GetString(),
-                ProfileBackgroundImageUrl =  profileBannerURL,
+                ProfileBackgroundImageUrl = res.RootElement.GetProperty("data").GetProperty("profile_image_url").GetString(), //for now
-                ProfileBannerURL = profileBannerURL,
+                ProfileBannerURL = res.RootElement.GetProperty("data").GetProperty("profile_image_url").GetString(), //for now
-                Protected = false, //res.RootElement.GetProperty("data").GetProperty("protected").GetBoolean(), 
+                Protected = res.RootElement.GetProperty("data").GetProperty("protected").GetBoolean(), 
             };
-
         }
 
+
         public bool IsUserApiRateLimited()
         {
+            // Retrieve limit from tooling
+            //_twitterAuthenticationInitializer.EnsureAuthenticationIsInitialized();
+            //ExceptionHandler.SwallowWebExceptions = false;
+            //RateLimit.RateLimitTrackerMode = RateLimitTrackerMode.TrackOnly;
+
+            //try
+            //{
+            //    var queryRateLimits = RateLimit.GetQueryRateLimit("https://api.twitter.com/1.1/users/show.json?screen_name=mastodon");
+
+            //    if (queryRateLimits != null)
+            //    {
+            //        return queryRateLimits.Remaining <= 0;
+            //    }
+            //}
+            //catch (Exception e)
+            //{
+            //    _logger.LogError(e, "Error retrieving rate limits");
+            //}
+
+            //// Fallback
+            //var currentCalls = _statisticsHandler.GetCurrentUserCalls();
+            //var maxCalls = _statisticsHandler.GetStatistics().UserCallsMax;
+            //return currentCalls >= maxCalls;
             return false;
         }
     }

src/BirdsiteLive.Wikidata/BirdsiteLive.Wikidata.csproj

@@ -1,18 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <OutputType>Exe</OutputType>
-        <TargetFramework>net7.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
-      <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
-      <Content Include="query.sparql">
-        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-      </Content>
-    </ItemGroup>
-
-</Project>

src/BirdsiteLive.Wikidata/Program.cs

@@ -1,48 +0,0 @@
-using System;
-using System.IO;
-using System.Net.Http;
-using System.Threading.Tasks;
-using System.Xml;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.DAL.Postgres.DataAccessLayers;
-using BirdsiteLive.DAL.Postgres.Settings;
-
-var settings = new PostgresSettings()
-{
-    ConnString = System.Environment.GetEnvironmentVariable("ConnString"),
-};
-var dal = new TwitterUserPostgresDal(settings);
-
-var twitterUser = new HashSet<string>();
-var twitterUserQuery = await dal.GetAllTwitterUsersAsync();
-Console.WriteLine("Loading twitter users");
-foreach (SyncTwitterUser user in twitterUserQuery)
-{
-    twitterUser.Add(user.Acct);
-}
-Console.WriteLine("Done loading twitter users");
-
-Console.WriteLine("Hello, World!");
-var client = new HttpClient();
-string query = new StreamReader("query.sparql").ReadToEnd();
-
-client.DefaultRequestHeaders.Add("Accept", "text/csv");
-client.DefaultRequestHeaders.Add("User-Agent", "BirdMakeup/1.0 (https://bird.makeup; coolbot@example.org) BirdMakeup/1.0");
-var response = await client.GetAsync($"https://query.wikidata.org/sparql?query={Uri.EscapeDataString(query)}");
-var content = await response.Content.ReadAsStringAsync();
-
-// Console.WriteLine(content);
-
-foreach (string n in content.Split("\n"))
-{
-    var s = n.Split(",");
-    if (n.Length < 2)
-        continue;
-    
-    var acct = s[1].ToLower();
-    var fedi = "@" + s[2];
-    await dal.UpdateTwitterUserFediAcctAsync(acct, fedi);
-    if (twitterUser.Contains(acct))
-        Console.WriteLine(fedi);
-}
-

src/BirdsiteLive.Wikidata/README.md

@@ -1,33 +0,0 @@
-# Wikidata service
-
-Wikidata is the metadata community behind Wikipedia. See for example
-[Hank Green](https://www.wikidata.org/wiki/Q550996). In his page, there are 
-all the links to his wikipedia pages, and many facts about him. What is 
-particularly useful to us is the twitter username (P2002) and mastodon 
-username (P4033). 
-
-From this information, we can build a feature that suggests to follow the 
-native fediverse account of someone you are trying to follow from Twitter.
-
-The main downside is that those redirect are only for somewhat famous
-people/organisations.
-
-## Goals
-### Being reusable by others
-All this data can be useful to many other fediverse projects: tools 
-for finding interesting accounts to follow, "verified" badge powered by 
-Wikipedia, etc. I hope that by working on improving this dataset, we can
-help other projects thrive. 
-### Being independent of Twitter
-Bird.makeup has to build features in a way that can't be suddenly cut off.
-Building this feature with a "Log in with Twitter" is not viable.
-Wikipedia is independent and outside of Elon's reach.
-
-Also this system supports many other services: TikTok, Reddit, YouTube, etc. 
-Which is really useful to expend the scope of this project while reusing as 
-much work as possible
-### Having great moderation
-
-Wikipedia has many tools to help curate data and remove troll's submissions,
-far better than anything I can build. I much prefer contribute to what
-they are doing than try to compete

src/BirdsiteLive.Wikidata/query.sparql

@@ -1,9 +0,0 @@
-#Cats
-SELECT ?item ?username ?username2 ?linkcount ?itemLabel
-WHERE 
-{
-  ?item wdt:P2002 ?username. 
-  ?item wdt:P4033 ?username2. 
-        ?item wikibase:sitelinks ?linkcount .
-  SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } # Helps get the label in your language, if not, then en language
-} ORDER BY DESC(?linkcount) LIMIT 5000

src/BirdsiteLive.sln

@@ -47,15 +47,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Moderation.Tes
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Common.Tests", "Tests\BirdsiteLive.Common.Tests\BirdsiteLive.Common.Tests.csproj", "{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Twitter.Tests", "Tests\BirdsiteLive.Twitter.Tests\BirdsiteLive.Twitter.Tests.csproj", "{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BSLManager", "BSLManager\BSLManager.csproj", "{4A84D351-E91B-4E58-8E20-211F0F4991D7}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Wikidata", "BirdsiteLive.Wikidata\BirdsiteLive.Wikidata.csproj", "{EAB43087-359C-46BD-8796-5F7D9B473B39}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BSLManager.Tests", "Tests\BSLManager.Tests\BSLManager.Tests.csproj", "{D4457271-620E-465A-B08E-7FC63C99A2F6}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SocialNetworks", "SocialNetworks", "{7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dotMakeup.HackerNews", "dotMakeup.HackerNews\dotMakeup.HackerNews.csproj", "{060DE3F7-DB7E-45FD-B233-104C3C464F57}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dotMakeup.HackerNews.Tests", "Tests\dotMakeup.HackerNews.Tests\dotMakeup.HackerNews.Tests.csproj", "{6D650384-7BDD-4628-A46C-2FE4A688DBA4}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -135,28 +129,21 @@ Global
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{160AD138-4E29-4706-8546-9826B529E9B2} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
+		{77C559D1-80A2-4B1C-A566-AE2D156944A4} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
 		{E64E7501-5DB8-4620-BA35-BA59FD746ABA} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
 		{155D46A4-2D05-47F2-8FFC-0B7C412A7652} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{D48450EE-D8BD-4228-9864-043AC88F7EE0} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
@@ -172,11 +159,7 @@ Global
 		{4BE541AC-8A93-4FA3-98AC-956CC2D5B748} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
 		{0A311BF3-4FD9-4303-940A-A3778890561C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
-		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
+		{D4457271-620E-465A-B08E-7FC63C99A2F6} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
-		{EAB43087-359C-46BD-8796-5F7D9B473B39} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57} = {7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}
-		{77C559D1-80A2-4B1C-A566-AE2D156944A4} = {7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE}

src/BirdsiteLive/BirdsiteLive.csproj

@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>1.0</Version>
+    <Version>0.20.0</Version>
-    <ContainerImageName>cloutier/bird.makeup</ContainerImageName>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="5.0.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.16.0" />
-    <PackageReference Include="Microsoft.NET.Build.Containers" Version="0.3.2" />
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.8" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.3" />
   </ItemGroup>
 
   <ItemGroup>
@@ -23,4 +23,7 @@
     <ProjectReference Include="..\BirdsiteLive.Twitter\BirdsiteLive.Twitter.csproj" />
     <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
   </ItemGroup>
+
+
+
 </Project>

src/BirdsiteLive/Component/NodeInfoViewComponent.cs

@@ -7,6 +7,7 @@ using BirdsiteLive.Domain.Repository;
 using BirdsiteLive.Services;
 using BirdsiteLive.Statistics.Domain;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Razor.Language.Intermediate;
 
 namespace BirdsiteLive.Component
 {
@@ -36,7 +37,7 @@ namespace BirdsiteLive.Component
                                       twitterAccountPolicy == ModerationTypeEnum.BlackListing,
                 WhitelistingEnabled = followerPolicy == ModerationTypeEnum.WhiteListing ||
                                       twitterAccountPolicy == ModerationTypeEnum.WhiteListing,
-                SyncLag = statistics.SyncLag
+                InstanceSaturation = statistics.Saturation
             };
             
             //viewModel = new NodeInfoViewModel
@@ -54,6 +55,5 @@ namespace BirdsiteLive.Component
         public bool BlacklistingEnabled { get; set; }
         public bool WhitelistingEnabled { get; set; }
         public int InstanceSaturation { get; set; }
-        public TimeSpan SyncLag { get; set; }
     }
 }

src/BirdsiteLive/Controllers/AboutController.cs

@@ -27,6 +27,18 @@ namespace BirdsiteLive.Controllers
             return View(stats);
         }
 
+        public IActionResult Blacklisting()
+        {
+            var status = GetModerationStatus();
+            return View("Blacklisting", status);
+        }
+
+        public IActionResult Whitelisting()
+        {
+            var status = GetModerationStatus();
+            return View("Whitelisting", status);
+        }
+
         private ModerationStatus GetModerationStatus()
         {
             var status = new ModerationStatus

src/BirdsiteLive/Controllers/DebugingController.cs

@@ -10,6 +10,7 @@ using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Domain;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.ViewFeatures;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {

src/BirdsiteLive/Controllers/UsersController.cs

@@ -2,16 +2,14 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text.Json;
 using System.Net.Mime;
 using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Models;
 using BirdsiteLive.Tools;
@@ -21,30 +19,27 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
+using Newtonsoft.Json;
 
 namespace BirdsiteLive.Controllers
 {
     public class UsersController : Controller
     {
-        private readonly ICachedTwitterUserService _twitterUserService;
+        private readonly ITwitterUserService _twitterUserService;
-        private readonly ICachedTwitterTweetsService _twitterTweetService;
+        private readonly ITwitterTweetsService _twitterTweetService;
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
         private readonly InstanceSettings _instanceSettings;
-        private readonly IFollowersDal _followersDal;
-        private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, IFollowersDal followersDal, ITwitterUserDal twitterUserDal, ILogger<UsersController> logger)
+        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
             _statusService = statusService;
             _instanceSettings = instanceSettings;
             _twitterTweetService = twitterTweetService;
-            _followersDal = followersDal;
-            _twitterUserDal = twitterUserDal;
             _logger = logger;
         }
         #endregion
@@ -64,7 +59,7 @@ namespace BirdsiteLive.Controllers
         [Route("/@{id}")]
         [Route("/users/{id}")]
         [Route("/users/{id}/remote_follow")]
-        public async Task<IActionResult> Index(string id)
+        public IActionResult Index(string id)
         {
             _logger.LogTrace("User Index: {Id}", id);
 
@@ -80,7 +75,7 @@ namespace BirdsiteLive.Controllers
             {
                 try
                 {
-                    user = await _twitterUserService.GetUserAsync(id);
+                    user = _twitterUserService.GetUser(id);
                 }
                 catch (UserNotFoundException)
                 {
@@ -116,7 +111,7 @@ namespace BirdsiteLive.Controllers
                     if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 };
                     if (notFound) return NotFound();
                     var apUser = _userService.GetUser(user);
-                    var jsonApUser = System.Text.Json.JsonSerializer.Serialize(apUser);
+                    var jsonApUser = JsonConvert.SerializeObject(apUser);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
@@ -124,12 +119,6 @@ namespace BirdsiteLive.Controllers
             if (isSaturated) return View("ApiSaturated");
             if (notFound) return View("UserNotFound");
 
-            Follower[] followers = new Follower[] { };
-            
-            var userDal = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
-            if (userDal != null)
-                followers = await _followersDal.GetFollowersAsync(userDal.Id);
-
             var displayableUser = new DisplayTwitterUser
             {
                 Name = user.Name,
@@ -138,9 +127,7 @@ namespace BirdsiteLive.Controllers
                 Url = user.Url,
                 ProfileImageUrl = user.ProfileImageUrl,
                 Protected = user.Protected,
-                FollowerCount = followers.Length,
+
-                MostPopularServer = followers.GroupBy(x => x.Host).OrderByDescending(x => x.Count()).Select(x => x.Key).FirstOrDefault("N/A"),
-                FediverseAccount = userDal.FediAcct,
                 InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}"
             };
             return View(displayableUser);
@@ -148,96 +135,31 @@ namespace BirdsiteLive.Controllers
 
         [Route("/@{id}/{statusId}")]
         [Route("/users/{id}/statuses/{statusId}")]
-        public async Task<IActionResult> Tweet(string id, string statusId)
+        public IActionResult Tweet(string id, string statusId)
         {
             var acceptHeaders = Request.Headers["Accept"];
-            if (!long.TryParse(statusId, out var parsedStatusId))
-                return NotFound();
-
-            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
-            if (tweet == null)
-                return NotFound();
-
-            if (tweet.Author.Acct != id)
-                return NotFound();
-            
-            var status = _statusService.GetStatus(id, tweet);
-
             if (acceptHeaders.Any())
             {
                 var r = acceptHeaders.First();
-
                 if (r.Contains("application/activity+json"))
                 {
-                    var jsonApUser = JsonSerializer.Serialize(status);
+                    if (!long.TryParse(statusId, out var parsedStatusId))
+                        return NotFound();
+
+                    var tweet = _twitterTweetService.GetTweet(parsedStatusId);
+                    if (tweet == null)
+                        return NotFound();
+
+                    //var user = _twitterService.GetUser(id);
+                    //if (user == null) return NotFound();
+
+                    var status = _statusService.GetStatus(id, tweet);
+                    var jsonApUser = JsonConvert.SerializeObject(status);
                     return Content(jsonApUser, "application/activity+json; charset=utf-8");
                 }
             }
 
-            //return Redirect($"https://twitter.com/{id}/status/{statusId}");
+            return Redirect($"https://twitter.com/{id}/status/{statusId}");
-            var displayTweet = new DisplayTweet 
-            {
-                Text = tweet.MessageContent,
-                OgUrl = $"https://twitter.com/{id}/status/{statusId}",
-                UserProfileImage = tweet.Author.ProfileImageUrl,
-                UserName = tweet.Author.Name,
-            };
-            return View(displayTweet);
-        }
-
-        // Mastodon API for QT in some apps
-        [Route("/api/v1/statuses/{statusId}")]
-        public async Task<IActionResult> mastoApi(string id, string statusId)
-        {
-            if (!long.TryParse(statusId, out var parsedStatusId))
-                return NotFound();
-
-            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
-            if (tweet == null)
-                return NotFound();
-            
-            var user = await _twitterUserService.GetUserAsync(tweet.Author.Acct);
-            var status = _statusService.GetActivity(tweet.Author.Acct, tweet);
-            var res = new MastodonPostApi()
-            {
-                id = parsedStatusId,
-                content = status.apObject.content,
-                created_at = status.published,
-                uri = $"https://{_instanceSettings.Domain}/users/{tweet.Author.Acct.ToLower()}/statuses/{tweet.Id}",
-                url = $"https://{_instanceSettings.Domain}/@{tweet.Author.Acct.ToLower()}/{tweet.Id}",
-                account = new MastodonUserApi()
-                {
-                    id = user.Id,
-                    username = user.Acct,
-                    acct = user.Acct,
-                    display_name = user.Name,
-                    note = user.Description,
-                    url = $"https://{_instanceSettings.Domain}/@{tweet.Author.Acct.ToLower()}",
-                    avatar = user.ProfileImageUrl,
-                    avatar_static = user.ProfileImageUrl,
-                    header = user.ProfileBannerURL,
-                    header_static = user.ProfileBannerURL,
-                }
-            };
-
-
-            var jsonApUser = JsonSerializer.Serialize(res);
-            return Content(jsonApUser, "application/activity+json; charset=utf-8");
-        }
-        [Route("/users/{id}/statuses/{statusId}/activity")]
-        public async Task<IActionResult> Activity(string id, string statusId)
-        {
-            if (!long.TryParse(statusId, out var parsedStatusId))
-                return NotFound();
-
-            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
-            if (tweet == null)
-                return NotFound();
-            
-            var status = _statusService.GetActivity(id, tweet);
-
-            var jsonApUser = JsonSerializer.Serialize(status);
-            return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
 
         [Route("/users/{id}/inbox")]
@@ -320,7 +242,7 @@ namespace BirdsiteLive.Controllers
             {
                 id = $"https://{_instanceSettings.Domain}/users/{id}/followers"
             };
-            var jsonApUser = JsonSerializer.Serialize(followers);
+            var jsonApUser = JsonConvert.SerializeObject(followers);
             return Content(jsonApUser, "application/activity+json; charset=utf-8");
         }
     }

src/BirdsiteLive/Controllers/WellKnownController.cs

@@ -142,7 +142,7 @@ namespace BirdsiteLive.Controllers
         }
 
         [Route("/.well-known/webfinger")]
-        public async Task<IActionResult> Webfinger(string resource = null)
+        public IActionResult Webfinger(string resource = null)
         {
             if (string.IsNullOrWhiteSpace(resource)) 
                 return BadRequest();
@@ -201,30 +201,26 @@ namespace BirdsiteLive.Controllers
             if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain)
                 return NotFound();
 
-            var user = await _twitterUserDal.GetTwitterUserAsync(name);
+            try
-            if (user is null)
             {
-                try
+                _twitterUserService.GetUser(name);
-                {
+            }
-                    await _twitterUserService.GetUserAsync(name);
+            catch (UserNotFoundException)
-                }
+            {
-                catch (UserNotFoundException)
+                return NotFound();
-                {
+            }
-                    return NotFound();
+            catch (UserHasBeenSuspendedException)
-                }
+            {
-                catch (UserHasBeenSuspendedException)
+                return NotFound();
-                {
+            }
-                    return NotFound();
+            catch (RateLimitExceededException)
-                }
+            {
-                catch (RateLimitExceededException)
+                return new ObjectResult("Too Many Requests") { StatusCode = 429 };
-                {
+            }
-                    return new ObjectResult("Too Many Requests") { StatusCode = 429 };
+            catch (Exception e)
-                }
+            {
-                catch (Exception e)
+                _logger.LogError(e, "Exception getting {Name}", name);
-                {
+                throw;
-                    _logger.LogError(e, "Exception getting {Name}", name);
-                    throw;
-                }
             }
 
             var actorUrl = UrlFactory.GetActorUrl(_settings.Domain, name);

src/BirdsiteLive/Models/DisplayTweet.cs

@@ -1,10 +0,0 @@
-namespace BirdsiteLive.Models
-{
-    public class DisplayTweet
-    {
-        public string Text { get; set; }
-        public string OgUrl { get; set; }
-        public string UserProfileImage { get; set; }
-        public string UserName { get; set; }
-    }
-}

src/BirdsiteLive/Models/DisplayTwitterUser.cs

@@ -8,10 +8,7 @@
         public string Url { get; set; }
         public string ProfileImageUrl { get; set; }
         public bool Protected { get; set; }
-        public int FollowerCount { get; set; }
-        public string MostPopularServer { get; set; }
 
         public string InstanceHandle { get; set; }
-        public string FediverseAccount { get; set; }
     }
 }

src/BirdsiteLive/Models/MastodonPostApi.cs

@@ -1,60 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace BirdsiteLive.Models;
-
-public class MastodonPostApi
-{
-    public long id { get; set; }
-    public string created_at { get; set; }
-    public long? in_reply_to_id { get; set; } = null;
-    public long? in_reply_to_account_id { get; set; } = null;
-    public bool sensitive { get; set; } = false;
-    public string spoiler_text { get; set; } = "";
-    public string visibility { get; set; } = "public";
-    public string language { get; set; } = "en";
-    public string uri { get; set; } 
-    public string url { get; set; } 
-    public int replies_count { get; set; } = 0;
-    public int reblogs_count { get; set; } = 0;
-    public int favorite_count { get; set; } = 0;
-    public string content { get; set; }
-    public MastodonUserApi account { get; set; }
-    public MastodonAppApi application { get; } = new MastodonAppApi();
-    
-    public List<MastodonAppApi> media_attachments { get; set; } = new List<MastodonAppApi>();
-    public List<MastodonAppApi> mentions { get; set; } = new List<MastodonAppApi>();
-    public List<MastodonAppApi> tags { get; set; } = new List<MastodonAppApi>();
-    public List<MastodonAppApi> emojis { get; set; } = new List<MastodonAppApi>();
-    public string card { get; set; }
-    public string poll { get; set; }
-    public string reblog { get; set; }
-}
-public class MastodonUserApi
-{
-    public long id { get; set; }
-    public string username { get; set; }
-    public string acct { get; set; }
-    public string display_name { get; set; }
-    public bool locked { get; set; } = false;
-    public bool bot { get; set; } = true;
-    public bool group { get; set; } = false;
-    public string note { get; set; }
-    public string url { get; set; }
-    public string avatar { get; set; }
-    public string avatar_static { get; set; }
-    public string header { get; set; }
-    public string header_static { get; set; }
-    public int followers_count { get; set; } = 0;
-    public int following_count { get; set; } = 0;
-    public int statuses_count { get; set; } = 0;
-
-    public List<MastodonAppApi> fields { get; set; } = new List<MastodonAppApi>();
-    public List<MastodonAppApi> emojis { get; set; } = new List<MastodonAppApi>();
-}
-
-public class MastodonAppApi
-{
-    public string name { get; set; } = "bird.makeup";
-    public string url { get; set; } = "https://bird.makeup/";
-}

src/BirdsiteLive/Properties/launchSettings.json

@@ -19,18 +19,16 @@
       "commandName": "Project",
       "launchBrowser": true,
       "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development",
+        "ASPNETCORE_ENVIRONMENT": "Development"
-        "Instance__ParallelTwitterRequests": "0"
       },
       "applicationUrl": "http://localhost:5000"
     },
     "Docker": {
       "commandName": "Docker",
       "launchBrowser": true,
-      "applicationUrl": "http://localhost:5000",
       "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}",
       "publishAllPorts": true,
-      "useSSL": false
+      "useSSL": true
     }
   }
 }

src/BirdsiteLive/Services/CachedStatisticsService.cs

@@ -13,55 +13,41 @@ namespace BirdsiteLive.Services
     public class CachedStatisticsService : ICachedStatisticsService
     {
         private readonly ITwitterUserDal _twitterUserDal;
-        private readonly IFollowersDal _followersDal;
 
-        private static Task<CachedStatistics> _cachedStatistics;
+        private static CachedStatistics _cachedStatistics;
         private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
-        public CachedStatisticsService(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings)
+        public CachedStatisticsService(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings)
         {
             _twitterUserDal = twitterUserDal;
             _instanceSettings = instanceSettings;
-            _followersDal = followersDal;
-            _cachedStatistics = CreateStats();
         }
         #endregion
 
         public async Task<CachedStatistics> GetStatisticsAsync()
         {
-            var stats = await _cachedStatistics;
+            if (_cachedStatistics == null ||
-            if ((DateTime.UtcNow - stats.RefreshedTime).TotalMinutes > 5)
+                (DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
             {
-                _cachedStatistics = CreateStats();
+                var twitterUserMax = _instanceSettings.MaxUsersCapacity;
+                var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
+                var saturation = (int)((double)twitterUserCount / twitterUserMax * 100);
+
+                _cachedStatistics = new CachedStatistics
+                {
+                    RefreshedTime = DateTime.UtcNow,
+                    Saturation = saturation
+                };
             }
 
-            return stats;
+            return _cachedStatistics;
-        }
-
-        private async Task<CachedStatistics> CreateStats()
-        {
-            var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
-            var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
-            var fediverseUsers = await _followersDal.GetFollowersCountAsync();
-
-            var stats = new CachedStatistics
-            {
-                RefreshedTime = DateTime.UtcNow,
-                SyncLag = twitterSyncLag,
-                TwitterUsers = twitterUserCount,
-                FediverseUsers = fediverseUsers
-            };
-        
-            return stats;
         }
     }
 
     public class CachedStatistics
     {
         public DateTime RefreshedTime { get; set; }
-        public TimeSpan SyncLag { get; set; }
+        public int Saturation { get; set; }
-        public int TwitterUsers { get; set; }
-        public int FediverseUsers { get; set; }
     }
 }

src/BirdsiteLive/Startup.cs

@@ -9,7 +9,6 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
 using BirdsiteLive.DAL.Postgres.Settings;
 using BirdsiteLive.Models;
-using BirdsiteLive.Services;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Tools;
 using Lamar;
@@ -56,6 +55,9 @@ namespace BirdsiteLive
 
         public void ConfigureContainer(ServiceRegistry services)
         {
+            var twitterSettings = Configuration.GetSection("Twitter").Get<TwitterSettings>();
+            services.For<TwitterSettings>().Use(x => twitterSettings);
+
             var instanceSettings = Configuration.GetSection("Instance").Get<InstanceSettings>();
             services.For<InstanceSettings>().Use(x => instanceSettings);
 
@@ -90,8 +92,6 @@ namespace BirdsiteLive
             services.For<ITwitterUserService>().Use<TwitterUserService>().Singleton();
 
             services.For<ITwitterAuthenticationInitializer>().Use<TwitterAuthenticationInitializer>().Singleton();
-            
-            services.For<ICachedStatisticsService>().Use<CachedStatisticsService>().Singleton();
 
             services.Scan(_ =>
             {

src/BirdsiteLive/Views/About/Blacklisting.cshtml

@@ -0,0 +1,27 @@
+@using BirdsiteLive.Domain.Repository
+@model BirdsiteLive.Controllers.ModerationStatus
+@{
+    ViewData["Title"] = "Blacklisting";
+}
+
+<div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
+    <h2>Blacklisting</h2>
+
+    @if (Model.Followers == ModerationTypeEnum.BlackListing)
+    {
+        <p><br />This node is blacklisting some instances and/or Fediverse users.<br /><br /></p>
+    }
+
+    @if (Model.TwitterAccounts == ModerationTypeEnum.BlackListing)
+    {
+        <p><br />This node is blacklisting some twitter users.<br /><br /></p>
+    }
+
+    @if (Model.Followers != ModerationTypeEnum.BlackListing && Model.TwitterAccounts != ModerationTypeEnum.BlackListing)
+    {
+        <p><br />This node is not using blacklisting.<br /><br /></p>
+    }
+
+    @*<h2>FAQ</h2>
+    <p>TODO</p>*@
+</div>

src/BirdsiteLive/Views/About/Index.cshtml

@@ -4,12 +4,27 @@
 }
 
 <div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
-    <h2>Service load</h2>
+    <h2>Node Saturation</h2>
 
     <p>
         <br/>
-        There are @Model.FediverseUsers fediverse users following @Model.TwitterUsers twitter users<br/>
+        This node usage is at @Model.Saturation%<br/>
         <br/>
     </p>
 
+    <h2>FAQ</h2>
+    <h4>Why is there a limit on the node?</h4>
+
+    <p>BirdsiteLIVE rely on the Twitter API to provide high quality content. This API has limitations and therefore limits node capacity.</p>
+
+    <h4>What happen when the node is saturated?</h4>
+
+    <p>
+        When the saturation rate goes above 100% the node will no longer update all accounts every 15 minutes and instead will reduce the pooling rate to stay under the API limits, the more saturated a node is the less efficient it will be.<br />
+        The software doesn't scale, and it's by design.
+    </p>
+
+    <h4>How can I reduce the node's saturation?</h4>
+
+    <p>If you're not on your own node, be reasonable and don't follow too much accounts. And if you can, host your own node. BirdsiteLIVE doesn't require a lot of resources to work and therefore is really cheap to self-host.</p>
 </div>

src/BirdsiteLive/Views/About/Whitelisting.cshtml

@@ -0,0 +1,27 @@
+@using BirdsiteLive.Domain.Repository
+@model BirdsiteLive.Controllers.ModerationStatus
+@{
+    ViewData["Title"] = "Whitelisting";
+}
+
+<div class="col-12 col-sm-12 col-md-10 col-lg-8 mx-auto">
+    <h2>Whitelisting</h2>
+
+    @if (Model.Followers == ModerationTypeEnum.WhiteListing)
+    {
+        <p><br />This node is whitelisting some instances and/or Fediverse users.<br /><br /></p>
+    }
+
+    @if (Model.TwitterAccounts == ModerationTypeEnum.WhiteListing)
+    {
+        <p><br />This node is whitelisting some twitter users.<br /><br /></p>
+    }
+
+    @if (Model.Followers != ModerationTypeEnum.WhiteListing && Model.TwitterAccounts != ModerationTypeEnum.WhiteListing)
+    {
+        <p><br />This node is not using whitelisting.<br /><br /></p>
+    }
+
+    @*<h2>FAQ</h2>
+    <p>TODO</p>*@
+</div>

src/BirdsiteLive/Views/Home/Index.cshtml

@@ -7,26 +7,23 @@
     <h1 class="display-4">Welcome</h1>
     <p>
         <br />
-        bird.makeup is a Twitter to ActivityPub bridge.<br />
+        BirdsiteLIVE is a Twitter to ActivityPub bridge.<br />
         Find a Twitter account below:
     </p>
     
     <form method="POST">
+        @*<div class="form-group">
+            <label for="exampleInputEmail1">Email address</label>
+            <input type="email" class="form-control" id="exampleInputEmail1" aria-describedby="emailHelp" placeholder="Enter email">
+            <small id="emailHelp" class="form-text text-muted">We'll never share your email with anyone else.</small>
+        </div>*@
         <div class="form-group">
+            @*<label for="exampleInputPassword1">Password</label>*@
             <input type="text" class="form-control col-8 col-sm-8 col-md-6 col-lg-4 mx-auto" id="handle" name="handle" autocomplete="off" placeholder="Twitter Handle">
         </div>
         <button type="submit" class="btn btn-primary">Show</button>
     </form>
 
-    <p style = "padding-top: 100px;">
-        <br />
-        bird.makeup is made with ❤️ by <a href="https://social.librem.one/@@vincent"> Vincent Cloutier</a> in 🇨🇦
-        <br />
-        <br />
-        Many thanks to our top Patreon supporters: <br/>
-        <a href="https://mstdn-social.com/@@fishcharlie">Charlie Fish</a>
-        
-    </p>
 
     @*@if (HtmlHelperExtensions.IsDebug())
     {

src/BirdsiteLive/Views/Shared/Components/NodeInfo/Default.cshtml

@@ -1,10 +1,22 @@
 @model BirdsiteLive.Component.NodeInfoViewModel
 
 <div>
+    @if (ViewData.Model.WhitelistingEnabled)
+    {
+        <a asp-controller="About" asp-action="Whitelisting" class="badge badge-light" title="What does this mean?">Whitelisting Enabled</a>
+    }
+    @if (ViewData.Model.BlacklistingEnabled)
+    {
+        <a asp-controller="About" asp-action="Blacklisting" class="badge badge-light" title="What does this mean?">Blacklisting Enabled</a>
+    }
+
     <div class="node-progress-bar">
-        <div class="node-progress-bar__label">
+        <div class="node-progress-bar__label"><a asp-controller="About" asp-action="Index">Instance saturation:</a></div>
-            <a asp-controller="About" asp-action="Index">Service load:</a>
+        <div class="progress node-progress-bar__bar">
-            @Math.Ceiling(ViewData.Model.SyncLag.TotalMinutes) minutes to fetch all twitter users 
+            <div class="progress-bar 
-         </div>
+                 @((ViewData.Model.InstanceSaturation > 50 && ViewData.Model.InstanceSaturation < 75) ? "bg-warning ":"")
+                 @((ViewData.Model.InstanceSaturation > 75 && ViewData.Model.InstanceSaturation < 100) ? "bg-danger ":"")
+                 @((ViewData.Model.InstanceSaturation > 100) ? "bg-saturation-danger ":"")" style="width: @ViewData.Model.InstanceSaturation%">@ViewData.Model.InstanceSaturation%</div>
+        </div>
     </div>
 </div>

src/BirdsiteLive/Views/Shared/_Layout.cshtml

@@ -6,24 +6,6 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>@Configuration.GetSection("Instance")["Name"] - @ViewData["Title"]</title>
-    @if(ViewData["AlternateLink"] != null) 
-    {
-    <link href='@ViewData["AlternateLink"]' rel='alternate' type='application/activity+json'>
-    <meta content='@ViewData["AlternateLink"]' property="og:url" />
-    }
-    @if(ViewData["MetaDescription"] != null) 
-    {
-    <meta content='@ViewData["MetaDescription"]' name='description'>
-    <meta content='@ViewData["MetaDescription"]' property="og:description" />
-    }
-    @if(ViewData["MetaTitle"] != null) 
-    {
-    <meta content='@ViewData["MetaTitle"]' name='og:title'>
-    }
-    @if(ViewData["MetaImage"] != null) 
-    {
-    <meta content='@ViewData["MetaImage"]' property="og:image" />
-    }
     <link rel="stylesheet" href="~/lib/bootstrap/dist/css/bootstrap.min.css" />
     <link rel="stylesheet" href="~/css/site.css" />
     <link rel="stylesheet" href="~/css/birdsite.css" />
@@ -65,9 +47,9 @@
         </div>
         <div class="container">
 
-            <a href="https://sr.ht/~cloutier/bird.makeup/">Source Code</a> @*<a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>*@
+            <a href="https://github.com/NicolasConstant/BirdsiteLive">Github</a> @*<a asp-area="" asp-controller="Home" asp-action="Privacy">Privacy</a>*@
 
-            <a class="btn btn-primary" href="https://www.patreon.com/birddotmakeup" style="float: right; margin:10px;">Support us on Patreon</a>
+            <span style="float: right;">BirdsiteLIVE @System.Reflection.Assembly.GetEntryAssembly().GetName().Version.ToString(3)</span>
         </div>
     </footer>
     <script src="~/lib/jquery/dist/jquery.min.js"></script>