glenux-archive/cloutier--bird.makeup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Various optimisations

Browse source
This commit is contained in:
Vincent Cloutier 2022-12-27 12:15:10 -05:00
parent dfdcb77924
commit 7f772ca125
6 changed files with 23 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View file

@ -60,7 +60,7 @@ namespace BirdsiteLive.Pipeline.Processors
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
} }
await Task.Delay(2 * 1000); await Task.Delay(1000);
} }
return usersWtTweets.ToArray(); return usersWtTweets.ToArray();

2
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -42,7 +42,7 @@ namespace BirdsiteLive.Pipeline.Processors
//var splitUsers = users.Split(25).ToList(); //var splitUsers = users.Split(25).ToList();
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync();
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(maxUsersNumber);
var userCount = users.Any() ? Math.Min(users.Length, 25) : 1; var userCount = users.Any() ? Math.Min(users.Length, 25) : 1;
//var splitNumber = (int) Math.Ceiling(userCount / 15d); //var splitNumber = (int) Math.Ceiling(userCount / 15d);

2
src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs
View file

@ -39,7 +39,7 @@ namespace BirdsiteLive.Twitter.Tools
_logger = logger; _logger = logger;
aTimer = new System.Timers.Timer(); aTimer = new System.Timers.Timer();
aTimer.Interval = 900000; // 15 minutes aTimer.Interval = 5 * 60 * 1000;
aTimer.Elapsed += async (sender, e) => await RefreshCred(); aTimer.Elapsed += async (sender, e) => await RefreshCred();
aTimer.Start(); aTimer.Start();

13
src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs
View file

@ -86,6 +86,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
} }
} }
public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber)
{
var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id LIMIT @maxNumber";
using (var dbConnection = Connection)
{
dbConnection.Open();
var result = await dbConnection.QueryAsync<SyncTwitterUser>(query, new { maxNumber });
return result.ToArray();
}
}
public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber) public async Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber)
{ {
var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber";

1
src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs
View file

@ -9,6 +9,7 @@ namespace BirdsiteLive.DAL.Contracts
Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
Task<SyncTwitterUser> GetTwitterUserAsync(string acct); Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
Task<SyncTwitterUser> GetTwitterUserAsync(int id); Task<SyncTwitterUser> GetTwitterUserAsync(int id);
Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber); Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(); Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync); Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);

17
src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
View file

@ -39,7 +39,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync( .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users); .ReturnsAsync(users);
@ -82,7 +82,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync( .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users.ToArray()) .ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0])
@ -129,7 +129,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.SetupSequence(x => x.GetAllTwitterUsersAsync( .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(users.ToArray()) .ReturnsAsync(users.ToArray())
.ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0])
@ -143,13 +143,8 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object);
processor.WaitFactor = 2; processor.WaitFactor = 2;
var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
var t2 = Task.Run(async () =>
{
while (buffer.Count < 11)
await Task.Delay(50);
});
await Task.WhenAny(t, t2, Task.Delay(5000)); await Task.WhenAny(t, Task.Delay(5000));
#region Validations #region Validations
maxUsersNumberProviderMock.VerifyAll(); maxUsersNumberProviderMock.VerifyAll();
@ -177,7 +172,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync( .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers)))
.ReturnsAsync(new SyncTwitterUser[0]); .ReturnsAsync(new SyncTwitterUser[0]);
@ -214,7 +209,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict); var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock twitterUserDalMock
.Setup(x => x.GetAllTwitterUsersAsync( .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
It.Is<int>(y => y == maxUsers))) It.Is<int>(y => y == maxUsers)))
.Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception())); .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));