glenux-archive/cloutier--bird.makeup
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

tweak checkpointing of twitter fetches

Browse source
This commit is contained in:
Vincent Cloutier 2023-03-17 19:45:51 -04:00
parent 5dcb1199c7
commit 66e2ba9b06
1 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs
View file

@ -51,9 +51,10 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
index++; index++;
var t = Task.Run(async () => { var t = Task.Run(async () => {
var user = userWtData.User;
var now = DateTime.UtcNow;
try try
{ {
var user = userWtData.User;
var tweets = await RetrieveNewTweets(user); var tweets = await RetrieveNewTweets(user);
_logger.LogInformation(index + "/" + syncTwitterUsers.Count() + " Got " + tweets.Length + " tweets from user " + user.Acct + " " ); _logger.LogInformation(index + "/" + syncTwitterUsers.Count() + " Got " + tweets.Length + " tweets from user " + user.Acct + " " );
if (tweets.Length > 0 && user.LastTweetPostedId != -1) if (tweets.Length > 0 && user.LastTweetPostedId != -1)
@ -61,15 +62,13 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
userWtData.Tweets = tweets; userWtData.Tweets = tweets;
usersWtTweets.Add(userWtData); usersWtTweets.Add(userWtData);
} }
else if (tweets.Length > 0 && user.LastTweetPostedId == -1) else if (tweets.Length > 0)
{ {
var tweetId = tweets.Last().Id; var tweetId = tweets.Last().Id;
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now); await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
} }
else else
{ {
var now = DateTime.UtcNow;
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
} }
@ -77,7 +76,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
catch(Exception e) catch(Exception e)
{ {
_logger.LogError(e.Message); _logger.LogError(e.Message);
await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
} }
}); });
todo.Add(t); todo.Add(t);