glenux-archive/cloutier--bird.makeup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

RetrieveTwitterUsersProcessor tweaks 2

Browse source
This commit is contained in:
Vincent Cloutier 2023-04-02 11:10:14 -04:00
parent 46f7594e43
commit 000214043c
1 changed files with 17 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs
View file

@ -1,4 +1,5 @@
using System; using System;
using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -44,21 +45,27 @@ namespace BirdsiteLive.Pipeline.Processors
await Task.Delay(10000); await Task.Delay(10000);
} }
var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m); var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
var userCount = users.Any() ? Math.Min(users.Length, 200) : 1; var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList(); var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
foreach (var u in splitUsers) foreach (var users in splitUsers)
{ {
ct.ThrowIfCancellationRequested(); ct.ThrowIfCancellationRequested();
UserWithDataToSync[] toSync = await Task.WhenAll( List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
u.Select(async x => new UserWithDataToSync foreach (var u in users)
{ User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) } {
) var followers = await _followersDal.GetFollowersAsync(u.Id);
); toSync.Add( new UserWithDataToSync()
{
User = u,
Followers = followers
});
}
await twitterUsersBufferBlock.SendAsync(toSync, ct); await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
} }