glenux-archive/cloutier--bird.makeup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

made retwrieve followers more parallel

Browse source
This commit is contained in:
Vincent Cloutier 2023-03-18 20:48:23 -04:00
parent dd7786ce38
commit 0bc8b96ea5
2 changed files with 1 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/BirdsiteLive.Pipeline/Processors/RetrieveFollowersProcessor.cs
View file

@ -30,11 +30,6 @@ namespace BirdsiteLive.Pipeline.Processors
user.Followers = followers; user.Followers = followers;
}); });
todo.Add(t); todo.Add(t);
if (todo.Count(x => !x.IsCompleted) >= 25)
{
await Task.WhenAny(todo);
}
} }
await Task.WhenAll(todo); await Task.WhenAll(todo);

2
src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs
View file

@ -45,7 +45,7 @@ namespace BirdsiteLive.Pipeline
{ BoundedCapacity = 1, CancellationToken = ct }); { BoundedCapacity = 1, CancellationToken = ct });
var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions ); var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }); var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions); var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct }); var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions); var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);