glenux-archive/cloutier--bird.makeup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

added limit to user retrieval

Browse source
This commit is contained in:
Nicolas Constant 2021-01-22 21:23:27 -05:00
parent b4c01ad326
commit 89c041f332
No known key found for this signature in database
GPG key ID: 1E9F677FB01A5688
9 changed files with 81 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic; using System;
using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -45,7 +46,8 @@ namespace BirdsiteLive.Pipeline.Processors
else if (tweets.Length > 0 && user.LastTweetPostedId == -1) else if (tweets.Length > 0 && user.LastTweetPostedId == -1)
{ {
var tweetId = tweets.Last().Id; var tweetId = tweets.Last().Id;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId); var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, now);
} }
} }

13
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -4,6 +4,7 @@ using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow; using System.Threading.Tasks.Dataflow;
using BirdsiteLive.Common.Extensions; using BirdsiteLive.Common.Extensions;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models; using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Contracts;
@ -15,12 +16,16 @@ namespace BirdsiteLive.Pipeline.Processors
{ {
private readonly ITwitterUserDal _twitterUserDal; private readonly ITwitterUserDal _twitterUserDal;
private readonly ILogger<RetrieveTwitterUsersProcessor> _logger; private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
private readonly InstanceSettings _instanceSettings;
public int WaitFactor = 1000 * 60; //1 min public int WaitFactor = 1000 * 60; //1 min
private int StartUpWarming = 4;
#region Ctor #region Ctor
public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger<RetrieveTwitterUsersProcessor> logger) public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
{ {
_twitterUserDal = twitterUserDal; _twitterUserDal = twitterUserDal;
_instanceSettings = instanceSettings;
_logger = logger; _logger = logger;
} }
#endregion #endregion
@ -33,7 +38,11 @@ namespace BirdsiteLive.Pipeline.Processors
try try
{ {
var users = await _twitterUserDal.GetAllTwitterUsersAsync(); var maxUsers = StartUpWarming > 0
? _instanceSettings.MaxUsersCapacity / 4
: _instanceSettings.MaxUsersCapacity;
StartUpWarming--;
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsers);
var userCount = users.Any() ? users.Length : 1; var userCount = users.Any() ? users.Length : 1;
var splitNumber = (int) Math.Ceiling(userCount / 15d); var splitNumber = (int) Math.Ceiling(userCount / 15d);

6
src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs
View file

@ -1,4 +1,5 @@
using System.Linq; using System;
using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Contracts;
@ -23,7 +24,8 @@ namespace BirdsiteLive.Pipeline.Processors
var userId = userWithTweetsToSync.User.Id; var userId = userWithTweetsToSync.User.Id;
var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min(); var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min();
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync); var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now);
} }
} }
} }

6
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
View file

@ -62,15 +62,15 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
} }
} }
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync() public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber)
{ {
var query = $"SELECT * FROM {_settings.TwitterUserTableName}"; var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC LIMIT @maxNumber";
using (var dbConnection = Connection) using (var dbConnection = Connection)
{ {
dbConnection.Open(); dbConnection.Open();
var result = await dbConnection.QueryAsync<SyncTwitterUser>(query); var result = await dbConnection.QueryAsync<SyncTwitterUser>(query, new { maxNumber });
return result.ToArray(); return result.ToArray();
} }
} }

2
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
View file

@ -8,7 +8,7 @@ namespace BirdsiteLive.DAL.Contracts
{ {
Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
Task<SyncTwitterUser> GetTwitterUserAsync(string acct); Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(); Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync); Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync);
Task DeleteTwitterUserAsync(string acct); Task DeleteTwitterUserAsync(string acct);
Task<int> GetTwitterUsersCountAsync(); Task<int> GetTwitterUsersCountAsync();

2
src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs
View file

@ -111,7 +111,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
await dal.CreateTwitterUserAsync(acct, lastTweetId); await dal.CreateTwitterUserAsync(acct, lastTweetId);
} }
var result = await dal.GetAllTwitterUsersAsync(); var result = await dal.GetAllTwitterUsersAsync(1000);
Assert.AreEqual(1000, result.Length); Assert.AreEqual(1000, result.Length);
Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Id == default);
Assert.IsFalse(result[0].Acct == default); Assert.IsFalse(result[0].Acct == default);

4
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs
View file

@ -1,3 +1,4 @@
using System;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -54,7 +55,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Setup(x => x.UpdateTwitterUserAsync( .Setup(x => x.UpdateTwitterUserAsync(
It.Is<int>(y => y == user1.Id), It.Is<int>(y => y == user1.Id),
It.Is<long>(y => y == tweets.Last().Id), It.Is<long>(y => y == tweets.Last().Id),
It.Is<long>(y => y == tweets.Last().Id) It.Is<long>(y => y == tweets.Last().Id),
It.IsAny<DateTime>()
)) ))
.Returns(Task.CompletedTask); .Returns(Task.CompletedTask);
#endregion #endregion

57
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow; using System.Threading.Tasks.Dataflow;
using BirdsiteLive.Common.Settings;
using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Contracts;
using BirdsiteLive.DAL.Models; using BirdsiteLive.DAL.Models;
using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Pipeline.Processors;
@ -26,18 +27,23 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
new SyncTwitterUser(), new SyncTwitterUser(),
new SyncTwitterUser(), new SyncTwitterUser(),
}; };
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
};
#endregion #endregion
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync()) .Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
.ReturnsAsync(users); .ReturnsAsync(users);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
processor.WaitFactor = 10; processor.WaitFactor = 10;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None); processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -60,19 +66,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
for (var i = 0; i < 30; i++) for (var i = 0; i < 30; i++)
users.Add(new SyncTwitterUser()); users.Add(new SyncTwitterUser());
var settings = new InstanceSettings
{
MaxUsersCapacity = 100
};
#endregion #endregion
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync()) .SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
.ReturnsAsync(users.ToArray()) .ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0]); .ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
processor.WaitFactor = 2; processor.WaitFactor = 2;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None); processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -95,19 +107,25 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
for (var i = 0; i < 31; i++) for (var i = 0; i < 31; i++)
users.Add(new SyncTwitterUser()); users.Add(new SyncTwitterUser());
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
};
#endregion #endregion
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync()) .SetupSequence(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
.ReturnsAsync(users.ToArray()) .ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0]); .ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
processor.WaitFactor = 2; processor.WaitFactor = 2;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None); processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -126,18 +144,24 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
{ {
#region Stubs #region Stubs
var buffer = new BufferBlock<SyncTwitterUser[]>(); var buffer = new BufferBlock<SyncTwitterUser[]>();
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
};
#endregion #endregion
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync()) .Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
.ReturnsAsync(new SyncTwitterUser[0]); .ReturnsAsync(new SyncTwitterUser[0]);
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
processor.WaitFactor = 1; processor.WaitFactor = 1;
processor.GetTwitterUsersAsync(buffer, CancellationToken.None); processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -154,18 +178,24 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
{ {
#region Stubs #region Stubs
var buffer = new BufferBlock<SyncTwitterUser[]>(); var buffer = new BufferBlock<SyncTwitterUser[]>();
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
};
#endregion #endregion
#region Mocks #region Mocks
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync()) .Setup(x => x.GetAllTwitterUsersAsync(
It.Is<int>(y => y == settings.MaxUsersCapacity)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception())); .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
processor.WaitFactor = 10; processor.WaitFactor = 10;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
@ -185,6 +215,11 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var buffer = new BufferBlock<SyncTwitterUser[]>(); var buffer = new BufferBlock<SyncTwitterUser[]>();
var canTokenS = new CancellationTokenSource(); var canTokenS = new CancellationTokenSource();
canTokenS.Cancel(); canTokenS.Cancel();
var settings = new InstanceSettings
{
MaxUsersCapacity = 10
};
#endregion #endregion
#region Mocks #region Mocks
@ -192,7 +227,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>(); var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
#endregion #endregion
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, settings, loggerMock.Object);
processor.WaitFactor = 1; processor.WaitFactor = 1;
await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
} }

12
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic; using System;
using System.Collections.Generic;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Contracts;
@ -60,7 +61,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Setup(x => x.UpdateTwitterUserAsync( .Setup(x => x.UpdateTwitterUserAsync(
It.Is<int>(y => y == user.Id), It.Is<int>(y => y == user.Id),
It.Is<long>(y => y == tweet2.Id), It.Is<long>(y => y == tweet2.Id),
It.Is<long>(y => y == tweet2.Id) It.Is<long>(y => y == tweet2.Id),
It.IsAny<DateTime>()
)) ))
.Returns(Task.CompletedTask); .Returns(Task.CompletedTask);
#endregion #endregion
@ -123,7 +125,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Setup(x => x.UpdateTwitterUserAsync( .Setup(x => x.UpdateTwitterUserAsync(
It.Is<int>(y => y == user.Id), It.Is<int>(y => y == user.Id),
It.Is<long>(y => y == tweet3.Id), It.Is<long>(y => y == tweet3.Id),
It.Is<long>(y => y == tweet2.Id) It.Is<long>(y => y == tweet2.Id),
It.IsAny<DateTime>()
)) ))
.Returns(Task.CompletedTask); .Returns(Task.CompletedTask);
#endregion #endregion
@ -194,7 +197,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
.Setup(x => x.UpdateTwitterUserAsync( .Setup(x => x.UpdateTwitterUserAsync(
It.Is<int>(y => y == user.Id), It.Is<int>(y => y == user.Id),
It.Is<long>(y => y == tweet3.Id), It.Is<long>(y => y == tweet3.Id),
It.Is<long>(y => y == tweet2.Id) It.Is<long>(y => y == tweet2.Id),
It.IsAny<DateTime>()
)) ))
.Returns(Task.CompletedTask); .Returns(Task.CompletedTask);
#endregion #endregion