glenux-archive/cloutier--bird.makeup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

removed max users

Browse source
This commit is contained in:
Vincent Cloutier 2023-01-20 11:13:00 -05:00
parent c4e6414229
commit 4a7373ec07
7 changed files with 7 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/BirdsiteLive.Common/Settings/InstanceSettings.cs
View file

@ -7,7 +7,6 @@
public string AdminEmail { get; set; } public string AdminEmail { get; set; }
public bool ResolveMentionsInProfiles { get; set; } public bool ResolveMentionsInProfiles { get; set; }
public bool PublishReplies { get; set; } public bool PublishReplies { get; set; }
public int MaxUsersCapacity { get; set; }
public string UnlistedTwitterAccounts { get; set; } public string UnlistedTwitterAccounts { get; set; }
public string SensitiveTwitterAccounts { get; set; } public string SensitiveTwitterAccounts { get; set; }

5
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -8,7 +8,6 @@ using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Models;
using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
namespace BirdsiteLive.Pipeline.Processors namespace BirdsiteLive.Pipeline.Processors
@ -16,17 +15,15 @@ namespace BirdsiteLive.Pipeline.Processors
public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor
{ {
private readonly ITwitterUserDal _twitterUserDal; private readonly ITwitterUserDal _twitterUserDal;
private readonly IMaxUsersNumberProvider _maxUsersNumberProvider;
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger; private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
private static Random rng = new Random(); private static Random rng = new Random();
public int WaitFactor = 1000 * 60; //1 min public int WaitFactor = 1000 * 60; //1 min
#region Ctor #region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger<RetrieveTwitterUsersProcessor> logger) public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger<RetrieveTwitterUsersProcessor> logger)
{ {
_twitterUserDal = twitterUserDal; _twitterUserDal = twitterUserDal;
_maxUsersNumberProvider = maxUsersNumberProvider;
_logger = logger; _logger = logger;
} }
#endregion #endregion

49
src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs
View file

@ -1,49 +0,0 @@
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
namespace BirdsiteLive.Pipeline.Tools
{
public interface IMaxUsersNumberProvider
{
Task<int> GetMaxUsersNumberAsync();
}
public class MaxUsersNumberProvider : IMaxUsersNumberProvider
{
private readonly InstanceSettings _instanceSettings;
private readonly ITwitterUserDal _twitterUserDal;
private int _totalUsersCount = -1;
private int _warmUpIterations;
private const int WarmUpMaxCapacity = 200;
#region Ctor
public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal)
{
_instanceSettings = instanceSettings;
_twitterUserDal = twitterUserDal;
}
#endregion
public async Task<int> GetMaxUsersNumberAsync()
{
// Init data
if (_totalUsersCount == -1)
{
_totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync();
_warmUpIterations = (int)(_totalUsersCount / (float)WarmUpMaxCapacity);
}
// Return if warm up ended
if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity;
// Calculate warm up value
var maxUsers = _warmUpIterations > 0
? WarmUpMaxCapacity
: _instanceSettings.MaxUsersCapacity;
_warmUpIterations--;
return maxUsers;
}
}
}

1
src/BirdsiteLive/Component/NodeInfoViewComponent.cs
View file

@ -37,7 +37,6 @@ namespace BirdsiteLive.Component
twitterAccountPolicy == ModerationTypeEnum.BlackListing, twitterAccountPolicy == ModerationTypeEnum.BlackListing,
WhitelistingEnabled = followerPolicy == ModerationTypeEnum.WhiteListing || WhitelistingEnabled = followerPolicy == ModerationTypeEnum.WhiteListing ||
twitterAccountPolicy == ModerationTypeEnum.WhiteListing, twitterAccountPolicy == ModerationTypeEnum.WhiteListing,
InstanceSaturation = statistics.Saturation,
SyncLag = statistics.SyncLag SyncLag = statistics.SyncLag
}; };

4
src/BirdsiteLive/Services/CachedStatisticsService.cs
View file

@ -32,16 +32,13 @@ namespace BirdsiteLive.Services
if (_cachedStatistics == null || if (_cachedStatistics == null ||
(DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15) (DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
{ {
var twitterUserMax = _instanceSettings.MaxUsersCapacity;
var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync(); var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag(); var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
var saturation = (int)((double)twitterUserCount / twitterUserMax * 100);
var fediverseUsers = await _followersDal.GetFollowersCountAsync(); var fediverseUsers = await _followersDal.GetFollowersCountAsync();
_cachedStatistics = new CachedStatistics _cachedStatistics = new CachedStatistics
{ {
RefreshedTime = DateTime.UtcNow, RefreshedTime = DateTime.UtcNow,
Saturation = saturation,
SyncLag = twitterSyncLag, SyncLag = twitterSyncLag,
TwitterUsers = twitterUserCount, TwitterUsers = twitterUserCount,
FediverseUsers = fediverseUsers FediverseUsers = fediverseUsers
@ -56,7 +53,6 @@ namespace BirdsiteLive.Services
{ {
public DateTime RefreshedTime { get; set; } public DateTime RefreshedTime { get; set; }
public TimeSpan SyncLag { get; set; } public TimeSpan SyncLag { get; set; }
public int Saturation { get; set; }
public int TwitterUsers { get; set; } public int TwitterUsers { get; set; }
public int FediverseUsers { get; set; } public int FediverseUsers { get; set; }
} }

42
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View file

@ -7,7 +7,6 @@ using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models; using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Pipeline.Processors;
using BirdsiteLive.Pipeline.Tools;
using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Models;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting; using Microsoft.VisualStudio.TestTools.UnitTesting;
@ -33,11 +32,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion #endregion
#region Mocks #region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersWithFollowersAsync( .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
@ -47,7 +41,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
processor.WaitFactor = 10; processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -75,11 +69,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion #endregion
#region Mocks #region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync( .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
@ -93,7 +82,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
processor.WaitFactor = 2; processor.WaitFactor = 2;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -121,11 +110,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion #endregion
#region Mocks #region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync( .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
@ -139,7 +123,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
processor.WaitFactor = 2; processor.WaitFactor = 2;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -163,10 +147,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion #endregion
#region Mocks #region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
@ -177,7 +157,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
processor.WaitFactor = 1; processor.WaitFactor = 1;
var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None); var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -199,11 +179,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion #endregion
#region Mocks #region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersWithFollowersAsync( .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
@ -213,7 +188,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
processor.WaitFactor = 10; processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -238,17 +213,12 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
#endregion #endregion
#region Mocks #region Mocks
var maxUsersNumberProviderMock = new Mock<IMaxUsersNumberProvider>(MockBehavior.Strict);
maxUsersNumberProviderMock
.Setup(x => x.GetMaxUsersNumberAsync())
.ReturnsAsync(maxUsers);
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object);
processor.WaitFactor = 1; processor.WaitFactor = 1;
await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
} }

82
src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs
View file

@ -1,82 +0,0 @@
using System.Threading.Tasks;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.Pipeline.Tools;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
namespace BirdsiteLive.Pipeline.Tests.Tools
{
[TestClass]
public class MaxUsersNumberProviderTests
{
[TestMethod]
public async Task GetMaxUsersNumberAsync_WarmUp_Test()
{
#region Stubs
var settings = new InstanceSettings
{
MaxUsersCapacity = 1000
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(1000);
#endregion
var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object);
var result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(200, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(200, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(200, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(200, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(200, result);
result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(1000, result);
#region Validations
twitterUserDalMock.VerifyAll();
#endregion
}
[TestMethod]
public async Task GetMaxUsersNumberAsync_NoWarmUp_Test()
{
#region Stubs
var settings = new InstanceSettings
{
MaxUsersCapacity = 1000
};
#endregion
#region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUsersCountAsync())
.ReturnsAsync(199);
#endregion
var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object);
var result = await provider.GetMaxUsersNumberAsync();
Assert.AreEqual(1000, result);
#region Validations
twitterUserDalMock.VerifyAll();
#endregion
}
}
}