Merge pull request #58 from NicolasConstant/develop

0.9.0 PR

src/BirdsiteLive.Cryptography/MagicKey.cs

@@ -105,25 +105,6 @@ namespace BirdsiteLive.Cryptography
             return new MagicKey(JsonConvert.SerializeObject(RSAKeyParms.From(rsa.ExportParameters(true))));
         }
 
-        //public static async Task<MagicKey> KeyForAuthor(ASObject obj)
-        //{
-        //    var authorId = (string)obj["email"].FirstOrDefault()?.Primitive;
-        //    if (authorId == null)
-        //    {
-        //        authorId = obj["name"].FirstOrDefault()?.Primitive + "@" + new Uri(obj.Id).Host;
-        //    }
-
-        //    var domain = authorId.Split('@')[1];
-        //    var hc = new HttpClient();
-        //    var wf = JsonConvert.DeserializeObject<WebfingerResult>(await hc.GetStringAsync($"https://{domain}/.well-known/webfinger?resource=acct:{Uri.EscapeDataString(authorId)}"));
-        //    var link = wf.links.FirstOrDefault(a => a.rel == "magic-public-key");
-        //    if (link == null) return null;
-
-        //    if (!link.href.StartsWith("data:")) return null; // does this happen?
-
-        //    return new MagicKey(link.href.Split(new char[] { ',' }, 2)[1]);
-        //}
-
         public byte[] BuildSignedData(string data, string dataType, string encoding, string algorithm)
         {
             var sig = data + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(dataType)) + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(encoding)) + "." + _encodeBase64Url(Encoding.UTF8.GetBytes(algorithm));

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -18,42 +18,42 @@ namespace BirdsiteLive.Domain
     {
         Task<Actor> GetUser(string objectId);
         Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null);
-        Task<HttpStatusCode> PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
+        Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
             string targetInbox);
     }
 
     public class ActivityPubService : IActivityPubService
     {
         private readonly InstanceSettings _instanceSettings;
+        private readonly IHttpClientFactory _httpClientFactory;
         private readonly ICryptoService _cryptoService;
 
         #region Ctor
-        public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings)
+        public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanceSettings, IHttpClientFactory httpClientFactory)
         {
             _cryptoService = cryptoService;
             _instanceSettings = instanceSettings;
+            _httpClientFactory = httpClientFactory;
         }
         #endregion
 
         public async Task<Actor> GetUser(string objectId)
         {
-            using (var httpClient = new HttpClient())
+            var httpClient = _httpClientFactory.CreateClient();
-            {
+            httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
-                httpClient.DefaultRequestHeaders.Add("Accept", "application/json");
+            var result = await httpClient.GetAsync(objectId);
-                var result = await httpClient.GetAsync(objectId);
+            var content = await result.Content.ReadAsStringAsync();
-                var content = await result.Content.ReadAsStringAsync();
+            return JsonConvert.DeserializeObject<Actor>(content);
-                return JsonConvert.DeserializeObject<Actor>(content);
-            }
         }
 
-        public async Task<HttpStatusCode> PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
+        public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
         {
             var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username);
             var noteUri = UrlFactory.GetNoteUrl(_instanceSettings.Domain, username, noteId);
 
             var now = DateTime.UtcNow;
             var nowString = now.ToString("s") + "Z";
-            
+
             var noteActivity = new ActivityCreateNote()
             {
                 context = "https://www.w3.org/ns/activitystreams",
@@ -67,7 +67,7 @@ namespace BirdsiteLive.Domain
                 apObject = note
             };
 
-            return await PostDataAsync(noteActivity, targetHost, actor, targetInbox);
+            await PostDataAsync(noteActivity, targetHost, actor, targetInbox);
         }
 
         public async Task<HttpStatusCode> PostDataAsync<T>(T data, string targetHost, string actorUrl, string inbox = null)
@@ -85,7 +85,7 @@ namespace BirdsiteLive.Domain
 
             var signature = _cryptoService.SignAndGetSignatureHeader(date, actorUrl, targetHost, digest, usedInbox);
 
-            var client = new HttpClient();
+            var client = _httpClientFactory.CreateClient();
             var httpRequestMessage = new HttpRequestMessage
             {
                 Method = HttpMethod.Post,
@@ -101,9 +101,8 @@ namespace BirdsiteLive.Domain
             };
 
             var response = await client.SendAsync(httpRequestMessage);
+            response.EnsureSuccessStatusCode();
             return response.StatusCode;
         }
-
-       
     }
 }

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -4,6 +4,10 @@
     <TargetFramework>netstandard2.0</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\BirdsiteLive.ActivityPub\BirdsiteLive.ActivityPub.csproj" />
     <ProjectReference Include="..\BirdsiteLive.Cryptography\BirdsiteLive.Cryptography.csproj" />

src/BirdsiteLive.Domain/UserService.cs

@@ -144,7 +144,7 @@ namespace BirdsiteLive.Domain
                 }
             };
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject);
-            return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK;
+            return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
         }
 
         private string OnlyKeepRoute(string inbox, string host)
@@ -188,7 +188,7 @@ namespace BirdsiteLive.Domain
                 }
             };
             var result = await _activityPubService.PostDataAsync(acceptFollow, followerHost, activity.apObject.apObject);
-            return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK;
+            return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
         }
 
         private async Task<SignatureValidationResult> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders, string body)

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -6,7 +6,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="1.1.1" />
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
   </ItemGroup>
 

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -5,18 +5,21 @@ using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Contracts;
+using Microsoft.Extensions.Logging;
 
 namespace BirdsiteLive.Pipeline.Processors
 {
     public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
     {
         private readonly ITwitterUserDal _twitterUserDal;
+        private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
         private const int SyncPeriod = 15; //in minutes
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
+            _logger = logger;
         }
         #endregion
 
@@ -35,8 +38,7 @@ namespace BirdsiteLive.Pipeline.Processors
                 }
                 catch (Exception e)
                 {
-                    Console.WriteLine(e);
+                    _logger.LogError(e, "Failing retrieving Twitter Users.");
-                    //TODO handle error
                 }
 
                 await Task.Delay(SyncPeriod * 1000 * 60, ct);

src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs

@@ -13,6 +13,7 @@ using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Logging;
 using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Processors
@@ -21,12 +22,14 @@ namespace BirdsiteLive.Pipeline.Processors
     {
         private readonly ISendTweetsToInboxTask _sendTweetsToInboxTask;
         private readonly ISendTweetsToSharedInboxTask _sendTweetsToSharedInbox;
+        private readonly ILogger<SendTweetsToFollowersProcessor> _logger;
 
         #region Ctor
-        public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox)
+        public SendTweetsToFollowersProcessor(ISendTweetsToInboxTask sendTweetsToInboxTask, ISendTweetsToSharedInboxTask sendTweetsToSharedInbox, ILogger<SendTweetsToFollowersProcessor> logger)
         {
             _sendTweetsToInboxTask = sendTweetsToInboxTask;
             _sendTweetsToSharedInbox = sendTweetsToSharedInbox;
+            _logger = logger;
         }
         #endregion
 
@@ -61,8 +64,8 @@ namespace BirdsiteLive.Pipeline.Processors
                 }
                 catch (Exception e)
                 {
-                    Console.WriteLine(e);
+                    var follower = followersPerInstance.First();
-                    //TODO handle error
+                    _logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.SharedInboxRoute);
                 }
             }
         }
@@ -77,8 +80,7 @@ namespace BirdsiteLive.Pipeline.Processors
                 }
                 catch (Exception e)
                 {
-                    Console.WriteLine(e);
+                    _logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.InboxRoute);
-                    //TODO handle error
                 }
             }
         }

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs

@@ -47,12 +47,8 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
                 foreach (var tweet in tweetsToSend)
                 {
                     var note = _statusService.GetStatus(user.Acct, tweet);
-                    var result = await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
+                    await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
-
+                    syncStatus = tweet.Id;
-                    if (result == HttpStatusCode.Accepted || result == HttpStatusCode.OK)
-                        syncStatus = tweet.Id;
-                    else
-                        throw new Exception("Posting new note activity failed");
                 }
             }
             finally

src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs

@@ -19,7 +19,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
         private readonly IStatusService _statusService;
         private readonly IActivityPubService _activityPubService;
         private readonly IFollowersDal _followersDal;
-        
+
         #region Ctor
         public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal)
         {
@@ -48,13 +48,8 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
                 foreach (var tweet in tweetsToSend)
                 {
                     var note = _statusService.GetStatus(user.Acct, tweet);
-                    var result =
+                    await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
-                        await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
+                    syncStatus = tweet.Id;
-
-                    if (result == HttpStatusCode.Accepted || result == HttpStatusCode.OK)
-                        syncStatus = tweet.Id;
-                    else
-                        throw new Exception("Posting new note activity failed");
                 }
             }
             finally

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
+using Microsoft.Extensions.Logging;
 
 namespace BirdsiteLive.Pipeline
 {
@@ -19,29 +20,31 @@ namespace BirdsiteLive.Pipeline
         private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
         private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
         private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
-        
+        private readonly ILogger<StatusPublicationPipeline> _logger;
+
         #region Ctor
-        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor)
+        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger<StatusPublicationPipeline> logger)
         {
             _retrieveTweetsProcessor = retrieveTweetsProcessor;
             _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
             _retrieveFollowersProcessor = retrieveFollowersProcessor;
             _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
+            _logger = logger;
         }
         #endregion
 
         public async Task ExecuteAsync(CancellationToken ct)
         {
             // Create blocks 
-            var twitterUsersBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct});
+            var twitterUsersBufferBlock = new BufferBlock<SyncTwitterUser[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
             var retrieveTweetsBlock = new TransformBlock<SyncTwitterUser[], UserWithTweetsToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct));
             var retrieveTweetsBufferBlock = new BufferBlock<UserWithTweetsToSync[]>(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct });
             var retrieveFollowersBlock = new TransformManyBlock<UserWithTweetsToSync[], UserWithTweetsToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct));
             var retrieveFollowersBufferBlock = new BufferBlock<UserWithTweetsToSync>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
-            var sendTweetsToFollowersBlock = new ActionBlock<UserWithTweetsToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct});
+            var sendTweetsToFollowersBlock = new ActionBlock<UserWithTweetsToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct });
 
             // Link pipeline
-            twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions {PropagateCompletion = true});
+            twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBlock.LinkTo(retrieveTweetsBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true });
             retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true });
@@ -51,12 +54,10 @@ namespace BirdsiteLive.Pipeline
             var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct);
 
             // Wait
-            await Task.WhenAny(new []{ retrieveTwitterAccountsTask , sendTweetsToFollowersBlock.Completion});
+            await Task.WhenAny(new[] { retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion });
 
-            var foreground = Console.ForegroundColor;
+            var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : sendTweetsToFollowersBlock.Completion.Exception;
-            Console.ForegroundColor = ConsoleColor.Red;
+            _logger.LogCritical(ex, "An error occurred, pipeline stopped");
-            Console.WriteLine("An error occured, pipeline stopped");
-            Console.ForegroundColor = foreground;
         }
     }
 }

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -5,6 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="5.0.0" />
     <PackageReference Include="TweetinviAPI" Version="4.0.3" />
   </ItemGroup>
 

src/BirdsiteLive.Twitter/CachedTwitterService.cs

@@ -0,0 +1,52 @@
+using System;
+using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace BirdsiteLive.Twitter
+{
+    public class CachedTwitterService : ITwitterService
+    {
+        private readonly ITwitterService _twitterService;
+
+        private MemoryCache _userCache = new MemoryCache(new MemoryCacheOptions()
+        {
+            SizeLimit = 5000
+        });
+        private MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
+            .SetSize(1)//Size amount
+            //Priority on removing when reaching size limit (memory pressure)
+            .SetPriority(CacheItemPriority.High)
+            // Keep in cache for this time, reset time if accessed.
+            .SetSlidingExpiration(TimeSpan.FromHours(24))
+            // Remove from cache after this time, regardless of sliding expiration
+            .SetAbsoluteExpiration(TimeSpan.FromDays(30));
+
+        #region Ctor
+        public CachedTwitterService(ITwitterService twitterService)
+        {
+            _twitterService = twitterService;
+        }
+        #endregion
+
+        public TwitterUser GetUser(string username)
+        {
+            if (!_userCache.TryGetValue(username, out TwitterUser user))
+            {
+                user = _twitterService.GetUser(username);
+                _userCache.Set(username, user, _cacheEntryOptions);
+            }
+
+            return user;
+        }
+
+        public ExtractedTweet GetTweet(long statusId)
+        {
+            return _twitterService.GetTweet(statusId);
+        }
+
+        public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
+        {
+            return _twitterService.GetTimeline(username, nberTweets, fromTweetId);
+        }
+    }
+}

src/BirdsiteLive/BirdsiteLive.csproj

@@ -4,11 +4,11 @@
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-    <Version>0.8.1</Version>
+    <Version>0.9.0</Version>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="4.1.0" />
+    <PackageReference Include="Lamar.Microsoft.DependencyInjection" Version="5.0.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.16.0" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.8" />
     <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.3" />

src/BirdsiteLive/Startup.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Common.Structs;
@@ -8,6 +9,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
 using BirdsiteLive.DAL.Postgres.Settings;
 using BirdsiteLive.Models;
+using BirdsiteLive.Twitter;
 using Lamar;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
@@ -46,6 +48,8 @@ namespace BirdsiteLive
             }
 
             services.AddControllersWithViews();
+
+            services.AddHttpClient();
         }
 
         public void ConfigureContainer(ServiceRegistry services)
@@ -61,7 +65,7 @@ namespace BirdsiteLive
 
             var logsSettings = Configuration.GetSection("Logging").Get<LogsSettings>();
             services.For<LogsSettings>().Use(x => logsSettings);
-
+            
             if (string.Equals(dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
             {
                 var connString = $"Host={dbSettings.Host};Username={dbSettings.User};Password={dbSettings.Password};Database={dbSettings.Name}";
@@ -70,7 +74,7 @@ namespace BirdsiteLive
                     ConnString = connString
                 };
                 services.For<PostgresSettings>().Use(x => postgresSettings);
-
+                
                 services.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
                 services.For<IFollowersDal>().Use<FollowersPostgresDal>().Singleton();
                 services.For<IDbInitializerDal>().Use<DbInitializerPostgresDal>().Singleton();
@@ -79,6 +83,9 @@ namespace BirdsiteLive
             {
                 throw new NotImplementedException($"{dbSettings.Type} is not supported");
             }
+            
+            services.For<ITwitterService>().DecorateAllWith<CachedTwitterService>();
+            services.For<ITwitterService>().Use<TwitterService>().Singleton();
 
             services.Scan(_ =>
             {

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Processors;
+using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
 
@@ -31,9 +32,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersAsync())
                 .ReturnsAsync(users);
+
+            var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
             processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
             await Task.Delay(50);
@@ -58,9 +61,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersAsync())
                 .ReturnsAsync(new SyncTwitterUser[0]);
+
+            var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
             processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
             await Task.Delay(50);
@@ -84,9 +89,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersAsync())
                 .Throws(new Exception());
+
+            var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
             await Task.WhenAny(t, Task.Delay(50));
@@ -110,9 +117,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+            var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
             await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
         }
     }

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs

@@ -6,6 +6,7 @@ using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Pipeline.Processors;
 using BirdsiteLive.Pipeline.Processors.SubTasks;
 using BirdsiteLive.Twitter.Models;
+using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
 
@@ -67,9 +68,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<string>(y => y == host),
                     It.Is<Follower[]>(y => y.Length == 2)))
                 .Returns(Task.CompletedTask);
+
+            var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
             #endregion
 
-            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object);
+            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object);
             var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
 
             #region Validations
@@ -135,9 +138,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                         It.Is<Follower[]>(y => y.Length == 1)))
                     .Returns(Task.CompletedTask);
             }
+
+            var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
             #endregion
 
-            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object);
+            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object);
             var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
 
             #region Validations
@@ -208,9 +213,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                     It.Is<string>(y => y == host2),
                     It.Is<Follower[]>(y => y.Length == 1)))
                 .Throws(new Exception());
+
+            var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
             #endregion
 
-            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object);
+            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object);
             var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
 
             #region Validations
@@ -274,9 +281,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             }
 
             var sendTweetsToSharedInboxTaskMock = new Mock<ISendTweetsToSharedInboxTask>(MockBehavior.Strict);
+
+            var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
             #endregion
 
-            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object);
+            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object);
             var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
 
             #region Validations
@@ -341,9 +350,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             }
 
             var sendTweetsToSharedInboxTaskMock = new Mock<ISendTweetsToSharedInboxTask>(MockBehavior.Strict);
+
+            var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
             #endregion
 
-            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object);
+            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object);
             var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
 
             #region Validations
@@ -412,9 +423,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Throws(new Exception());
 
             var sendTweetsToSharedInboxTaskMock = new Mock<ISendTweetsToSharedInboxTask>(MockBehavior.Strict);
+
+            var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
             #endregion
 
-            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object);
+            var processor = new SendTweetsToFollowersProcessor(sendTweetsToInboxTaskMock.Object, sendTweetsToSharedInboxTaskMock.Object, loggerMock.Object);
             var result = await processor.ProcessAsync(userWithTweets, CancellationToken.None);
 
             #region Validations

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Net;
+using System.Net.Http;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.DAL.Contracts;
@@ -63,7 +64,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     It.Is<string>(y => y == tweetId.ToString()),
                     It.Is<string>(y => y == host),
                     It.Is<string>(y => y == inbox)))
-                .ReturnsAsync(HttpStatusCode.Accepted);
+                .Returns(Task.CompletedTask);
 
             var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
             statusServiceMock
@@ -136,7 +137,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                         It.Is<string>(y => y == tweetId.ToString()),
                         It.Is<string>(y => y == host),
                         It.Is<string>(y => y == inbox)))
-                    .ReturnsAsync(HttpStatusCode.Accepted);
+                    .Returns(Task.CompletedTask);
             }
 
             var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
@@ -168,7 +169,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
         }
 
         [TestMethod]
-        [ExpectedException(typeof(Exception))]
+        [ExpectedException(typeof(HttpRequestException))]
         public async Task ExecuteAsync_MultipleTweets_Error_Test()
         {
             #region Stubs
@@ -213,7 +214,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     It.Is<string>(y => y == tweetId2.ToString()),
                     It.Is<string>(y => y == host),
                     It.Is<string>(y => y == inbox)))
-                .ReturnsAsync(HttpStatusCode.Accepted);
+                .Returns(Task.CompletedTask);
 
             activityPubService
                 .Setup(x => x.PostNewNoteActivity(
@@ -222,7 +223,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     It.Is<string>(y => y == tweetId3.ToString()),
                     It.Is<string>(y => y == host),
                     It.Is<string>(y => y == inbox)))
-                .ReturnsAsync(HttpStatusCode.InternalServerError);
+                .Throws(new HttpRequestException());
 
             var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
             foreach (var tweetId in new[] { tweetId2, tweetId3 })

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Net.Http;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.DAL.Contracts;
@@ -81,7 +82,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     It.Is<string>(y => y == tweetId.ToString()),
                     It.Is<string>(y => y == host),
                     It.Is<string>(y => y == inbox)))
-                .ReturnsAsync(HttpStatusCode.Accepted);
+                .Returns(Task.CompletedTask);
 
             var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
             statusServiceMock
@@ -174,7 +175,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                         It.Is<string>(y => y == tweetId.ToString()),
                         It.Is<string>(y => y == host),
                         It.Is<string>(y => y == inbox)))
-                    .ReturnsAsync(HttpStatusCode.Accepted);
+                    .Returns(Task.CompletedTask);
             }
 
             var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
@@ -209,7 +210,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
         }
 
         [TestMethod]
-        [ExpectedException(typeof(Exception))]
+        [ExpectedException(typeof(HttpRequestException))]
         public async Task ExecuteAsync_MultipleTweets_Error_Test()
         {
             #region Stubs
@@ -271,7 +272,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     It.Is<string>(y => y == tweetId2.ToString()),
                     It.Is<string>(y => y == host),
                     It.Is<string>(y => y == inbox)))
-                .ReturnsAsync(HttpStatusCode.Accepted);
+                .Returns(Task.CompletedTask);
 
             activityPubService
                 .Setup(x => x.PostNewNoteActivity(
@@ -280,7 +281,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     It.Is<string>(y => y == tweetId3.ToString()),
                     It.Is<string>(y => y == host),
                     It.Is<string>(y => y == inbox)))
-                .ReturnsAsync(HttpStatusCode.InternalServerError);
+                .Throws(new HttpRequestException());
 
             var statusServiceMock = new Mock<IStatusService>(MockBehavior.Strict);
             foreach (var tweetId in new[] { tweetId2, tweetId3 })