glenux-archive/cloutier--bird.makeup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

reduced batch size

Browse source
This commit is contained in:
Vincent Cloutier 2022-12-25 14:41:28 -05:00
parent 97f982903e
commit e7438057d1
2 changed files with 4 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View file

@ -58,7 +58,7 @@ namespace BirdsiteLive.Pipeline.Processors
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
} }
await Task.Delay(10 * 1000); await Task.Delay(4 * 1000);
} }
return usersWtTweets.ToArray(); return usersWtTweets.ToArray();

17
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -38,29 +38,18 @@ namespace BirdsiteLive.Pipeline.Processors
try try
{ {
var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); var users = await _twitterUserDal.GetAllTwitterUsersAsync(50);
var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber);
var userCount = users.Any() ? users.Length : 1; var splitUsers = users.Split(50).ToList();
var splitNumber = (int) Math.Ceiling(userCount / 15d);
var splitUsers = users.Split(splitNumber).ToList();
foreach (var u in splitUsers) foreach (var u in splitUsers)
{ {
ct.ThrowIfCancellationRequested(); ct.ThrowIfCancellationRequested();
await twitterUsersBufferBlock.SendAsync(u.ToArray(), ct); await twitterUsersBufferBlock.SendAsync(u.ToArray(), ct);
await Task.Delay(WaitFactor, ct);
} }
var splitCount = splitUsers.Count(); await Task.Delay(1000, ct);
if (splitCount < 15) await Task.Delay((15 - splitCount) * WaitFactor, ct); //Always wait 15min
//// Extra wait time to fit 100.000/day limit
//var extraWaitTime = (int)Math.Ceiling((60 / ((100000d / 24) / userCount)) - 15);
//if (extraWaitTime < 0) extraWaitTime = 0;
//await Task.Delay(extraWaitTime * 1000, ct);
} }
catch (Exception e) catch (Exception e)
{ {