glenux-archive/cloutier--bird.makeup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

RetrieveTwitterUsersProcessor tweaks

Browse source
This commit is contained in:
Vincent Cloutier 2023-04-02 10:23:45 -04:00
parent 3346b7b5e8
commit 46f7594e43
1 changed files with 21 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -38,31 +38,31 @@ namespace BirdsiteLive.Pipeline.Processors
{ {
ct.ThrowIfCancellationRequested(); ct.ThrowIfCancellationRequested();
try if (_instanceSettings.ParallelTwitterRequests == 0)
{ {
var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m); while (true)
await Task.Delay(10000);
var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
foreach (var u in splitUsers)
{
ct.ThrowIfCancellationRequested();
UserWithDataToSync[] toSync = await Task.WhenAll(
u.Select(async x => new UserWithDataToSync
{ User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) }
)
);
await twitterUsersBufferBlock.SendAsync(toSync, ct);
}
await Task.Delay(10, ct); // this is somehow necessary
} }
catch (Exception e)
var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
foreach (var u in splitUsers)
{ {
_logger.LogError(e, "Failing retrieving Twitter Users."); ct.ThrowIfCancellationRequested();
UserWithDataToSync[] toSync = await Task.WhenAll(
u.Select(async x => new UserWithDataToSync
{ User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) }
)
);
await twitterUsersBufferBlock.SendAsync(toSync, ct);
} }
await Task.Delay(10, ct); // this is somehow necessary
} }
} }
} }