From 66e2ba9b060921d1660ea1bc3169e12538f9ea80 Mon Sep 17 00:00:00 2001 From: Vincent Cloutier Date: Fri, 17 Mar 2023 19:45:51 -0400 Subject: [PATCH] tweak checkpointing of twitter fetches --- .../Processors/RetrieveTweetsProcessor.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 119e1b4..4678bd3 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -51,9 +51,10 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks index++; var t = Task.Run(async () => { + var user = userWtData.User; + var now = DateTime.UtcNow; try { - var user = userWtData.User; var tweets = await RetrieveNewTweets(user); _logger.LogInformation(index + "/" + syncTwitterUsers.Count() + " Got " + tweets.Length + " tweets from user " + user.Acct + " " ); if (tweets.Length > 0 && user.LastTweetPostedId != -1) @@ -61,15 +62,13 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks userWtData.Tweets = tweets; usersWtTweets.Add(userWtData); } - else if (tweets.Length > 0 && user.LastTweetPostedId == -1) + else if (tweets.Length > 0) { var tweetId = tweets.Last().Id; - var now = DateTime.UtcNow; await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now); } else { - var now = DateTime.UtcNow; await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); } @@ -77,7 +76,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks catch(Exception e) { _logger.LogError(e.Message); - + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); } }); todo.Add(t);