Merge pull request #49 from NicolasConstant/topic_add-api-statistics

Topic add api statistics

src/BirdsiteLive.Domain/Statistics/ExtractionStatisticsHandler.cs

@@ -0,0 +1,80 @@
+using System.Threading;
+using System.Timers;
+
+namespace BirdsiteLive.Domain.Statistics
+{
+    public interface IExtractionStatisticsHandler
+    {
+        void ExtractedDescription(int mentionsCount);
+        void ExtractedStatus(int mentionsCount);
+        ExtractionStatistics GetStatistics();
+    }
+    
+    public class ExtractionStatisticsHandler : IExtractionStatisticsHandler
+    {
+        private static int _lastDescriptionMentionsExtracted;
+        private static int _lastStatusMentionsExtracted;
+
+        private static int _descriptionMentionsExtracted;
+        private static int _statusMentionsExtracted;
+        
+        private static System.Timers.Timer _resetTimer;
+
+        #region Ctor
+        public ExtractionStatisticsHandler()
+        {
+            if (_resetTimer == null)
+            {
+                _resetTimer = new System.Timers.Timer();
+                _resetTimer.Elapsed += OnTimeResetEvent;
+                _resetTimer.Interval = 24 * 60 * 60 * 1000; // 24h
+                _resetTimer.Enabled = true;
+            }
+        }
+        #endregion
+
+        private void OnTimeResetEvent(object sender, ElapsedEventArgs e)
+        {
+            _lastDescriptionMentionsExtracted = _descriptionMentionsExtracted;
+            _lastStatusMentionsExtracted = _statusMentionsExtracted;
+
+            // Reset
+            Interlocked.Exchange(ref _descriptionMentionsExtracted, 0);
+            Interlocked.Exchange(ref _statusMentionsExtracted, 0);
+        }
+
+        public void ExtractedDescription(int mentionsCount)
+        {
+            Interlocked.Increment(ref _descriptionMentionsExtracted);
+        }
+
+        public void ExtractedStatus(int mentionsCount)
+        {
+            Interlocked.Increment(ref _statusMentionsExtracted);
+        }
+
+        public ExtractionStatistics GetStatistics()
+        {
+            return new ExtractionStatistics(_descriptionMentionsExtracted, _statusMentionsExtracted, _lastDescriptionMentionsExtracted, _lastStatusMentionsExtracted);
+        }
+    }
+
+    public class ExtractionStatistics
+    {
+        #region Ctor
+        public ExtractionStatistics(int mentionsInDescriptionsExtraction, int mentionsInStatusesExtraction, int lastMentionsInDescriptionsExtraction, int lastMentionsInStatusesExtraction)
+        {
+            MentionsInDescriptionsExtraction = mentionsInDescriptionsExtraction;
+            MentionsInStatusesExtraction = mentionsInStatusesExtraction;
+            LastMentionsInDescriptionsExtraction = lastMentionsInDescriptionsExtraction;
+            LastMentionsInStatusesExtraction = lastMentionsInStatusesExtraction;
+        }
+        #endregion
+
+        public int MentionsInDescriptionsExtraction { get; }
+        public int MentionsInStatusesExtraction { get; }
+
+        public int LastMentionsInDescriptionsExtraction { get; }
+        public int LastMentionsInStatusesExtraction { get; }
+    }
+}

src/BirdsiteLive.Domain/StatusService.cs

@@ -7,6 +7,7 @@ using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Models;
@@ -23,12 +24,14 @@ namespace BirdsiteLive.Domain
     {
         private readonly InstanceSettings _instanceSettings;
         private readonly IStatusExtractor _statusExtractor;
+        private readonly IExtractionStatisticsHandler _statisticsHandler;
         
         #region Ctor
-        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor)
+        public StatusService(InstanceSettings instanceSettings, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
         {
             _instanceSettings = instanceSettings;
             _statusExtractor = statusExtractor;
+            _statisticsHandler = statisticsHandler;
         }
         #endregion
 
@@ -41,6 +44,7 @@ namespace BirdsiteLive.Domain
             var apPublic = "https://www.w3.org/ns/activitystreams#Public";
 
             var extractedTags = _statusExtractor.ExtractTags(tweet.MessageContent);
+            _statisticsHandler.ExtractedStatus(extractedTags.tags.Count(x => x.type == "Mention"));
 
             string inReplyTo = null;
             if (tweet.InReplyToStatusId != default)

src/BirdsiteLive.Domain/UserService.cs

@@ -10,6 +10,7 @@ using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Cryptography;
 using BirdsiteLive.Domain.BusinessUseCases;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Domain.Tools;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Core.Exceptions;
@@ -33,9 +34,10 @@ namespace BirdsiteLive.Domain
         private readonly ICryptoService _cryptoService;
         private readonly IActivityPubService _activityPubService;
         private readonly IStatusExtractor _statusExtractor;
+        private readonly IExtractionStatisticsHandler _statisticsHandler;
 
         #region Ctor
-        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor)
+        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler)
         {
             _instanceSettings = instanceSettings;
             _cryptoService = cryptoService;
@@ -43,6 +45,7 @@ namespace BirdsiteLive.Domain
             _processFollowUser = processFollowUser;
             _processUndoFollowUser = processUndoFollowUser;
             _statusExtractor = statusExtractor;
+            _statisticsHandler = statisticsHandler;
             //_host = $"https://{instanceSettings.Domain.Replace("https://",string.Empty).Replace("http://", string.Empty).TrimEnd('/')}";
         }
         #endregion
@@ -58,6 +61,8 @@ namespace BirdsiteLive.Domain
             {
                 var extracted = _statusExtractor.ExtractTags(description);
                 description = extracted.content;
+
+                _statisticsHandler.ExtractedDescription(extracted.tags.Count(x => x.type == "Mention"));
             }
 
             var user = new Actor

src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs

@@ -45,6 +45,10 @@ namespace BirdsiteLive.Twitter.Extractors
                     message = message.Replace("RT", "[RT]");
             }
 
+            // Expand URLs
+            foreach (var url in tweet.Urls.OrderByDescending(x => x.URL.Length))
+                message = message.Replace(url.URL, url.ExpandedURL);
+
             return message;
         }
 

src/BirdsiteLive.Twitter/Models/ApiStatistics.cs

@@ -2,11 +2,17 @@
 {
     public class ApiStatistics
     {
-        public int UserCallsCount { get; set; }
+        public int UserCallsCountMin { get; set; }
+        public int UserCallsCountAvg { get; set; }
+        public int UserCallsCountMax { get; set; }
         public int UserCallsMax { get; set; }
-        public int TweetCallsCount { get; set; }
+        public int TweetCallsCountMin { get; set; }
+        public int TweetCallsCountAvg { get; set; }
+        public int TweetCallsCountMax { get; set; }
         public int TweetCallsMax { get; set; }
-        public int TimelineCallsCount { get; set; }
+        public int TimelineCallsCountMin { get; set; }
+        public int TimelineCallsCountAvg { get; set; }
+        public int TimelineCallsCountMax { get; set; }
         public int TimelineCallsMax { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Statistics/TwitterStatisticsHandler.cs

@@ -1,4 +1,7 @@
-using System.Threading;
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Threading;
 using System.Timers;
 using BirdsiteLive.Twitter.Models;
 
@@ -15,14 +18,12 @@ namespace BirdsiteLive.Statistics.Domain
     //Rate limits: https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits
     public class TwitterStatisticsHandler : ITwitterStatisticsHandler
     {
-        private static int _previousUserCalls;
-        private static int _previousTweetCalls;
-        private static int _previousTimelineCalls;
-
         private static int _userCalls;
         private static int _tweetCalls;
         private static int _timelineCalls;
 
+        private static ConcurrentDictionary<DateTime, ApiStatisticsSnapshot> _snapshots = new ConcurrentDictionary<DateTime, ApiStatisticsSnapshot>();
+
         private static System.Timers.Timer _resetTimer;
 
         #region Ctor
@@ -40,13 +41,23 @@ namespace BirdsiteLive.Statistics.Domain
 
         private void OnTimeResetEvent(object sender, ElapsedEventArgs e)
         {
-            _previousUserCalls = _userCalls;
+            // Add snapshot
-            _previousTweetCalls = _tweetCalls;
+            var snapshot = new ApiStatisticsSnapshot(_userCalls, _tweetCalls, _timelineCalls);
-            _previousTimelineCalls = _timelineCalls;
+            bool success;
+            do
+            {
+                success = _snapshots.TryAdd(snapshot.SnapshotDate, snapshot);
+            } while (!success);
             
+            // Reset
             Interlocked.Exchange(ref _userCalls, 0);
             Interlocked.Exchange(ref _tweetCalls, 0);
             Interlocked.Exchange(ref _timelineCalls, 0);
+
+            // Clean up 
+            var now = DateTime.UtcNow;
+            var oldSnapshots = _snapshots.Keys.Where(x => (now - x).TotalHours > 24).ToList();
+            foreach (var old in oldSnapshots) _snapshots.TryRemove(old, out var data);
         }
 
         public void CalledUserApi()  //GET users/show - 900/15mins
@@ -66,15 +77,44 @@ namespace BirdsiteLive.Statistics.Domain
 
         public ApiStatistics GetStatistics()
         {
+            var snapshots = _snapshots.Values.ToList();
+            var userCalls = snapshots.Select(x => x.UserCalls).ToList();
+            var tweetCalls = snapshots.Select(x => x.TweetCalls).ToList();
+            var timelineCalls = snapshots.Select(x => x.TimelineCalls).ToList();
+
             return new ApiStatistics
             {
-                UserCallsCount = _previousUserCalls,
+                UserCallsCountMin = userCalls.Any() ? userCalls.Min() : 0,
+                UserCallsCountAvg = userCalls.Any() ? (int)userCalls.Average() : 0,
+                UserCallsCountMax = userCalls.Any() ? userCalls.Max() : 0,
                 UserCallsMax = 900,
-                TweetCallsCount = _previousTweetCalls,
+                TweetCallsCountMin = tweetCalls.Any() ? tweetCalls.Min() : 0,
+                TweetCallsCountAvg = tweetCalls.Any() ? (int)tweetCalls.Average() : 0,
+                TweetCallsCountMax = tweetCalls.Any() ? tweetCalls.Max() : 0,
                 TweetCallsMax = 300,
-                TimelineCallsCount = _previousTimelineCalls,
+                TimelineCallsCountMin = timelineCalls.Any() ? timelineCalls.Min() : 0,
+                TimelineCallsCountAvg = timelineCalls.Any() ? (int)timelineCalls.Average() : 0,
+                TimelineCallsCountMax = timelineCalls.Any() ? timelineCalls.Max() : 0,
                 TimelineCallsMax = 1500
             };
         }
     }
+
+    internal class ApiStatisticsSnapshot 
+    {
+        #region Ctor
+        public ApiStatisticsSnapshot(int userCalls, int tweetCalls, int timelineCalls)
+        {
+            UserCalls = userCalls;
+            TweetCalls = tweetCalls;
+            TimelineCalls = timelineCalls;
+            SnapshotDate = DateTime.UtcNow;
+        }
+        #endregion
+
+        public DateTime SnapshotDate { get;  }
+        public int UserCalls { get; set; }
+        public int TweetCalls { get; set; }
+        public int TimelineCalls { get; set; }
+    }
 }

src/BirdsiteLive.Twitter/TwitterService.cs

@@ -43,11 +43,16 @@ namespace BirdsiteLive.Twitter
             _statisticsHandler.CalledUserApi();
             if (user == null) return null;
 
+            // Expand URLs
+            var description = user.Description;
+            foreach (var descriptionUrl in user.Entities?.Description?.Urls?.OrderByDescending(x => x.URL.Length))
+                description = description.Replace(descriptionUrl.URL, descriptionUrl.ExpandedURL);
+
             return new TwitterUser
             {
                 Acct = username,
                 Name = user.Name,
-                Description = user.Description,
+                Description = description,
                 Url = $"https://twitter.com/{username}",
                 ProfileImageUrl = user.ProfileImageUrlFullSize,
                 ProfileBackgroundImageUrl = user.ProfileBackgroundImageUrlHttps,

src/BirdsiteLive/Controllers/StatisticsController.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Domain.Statistics;
 using BirdsiteLive.Statistics.Domain;
 using Microsoft.AspNetCore.Mvc;
 
@@ -13,13 +14,16 @@ namespace BirdsiteLive.Controllers
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
         private readonly ITwitterStatisticsHandler _twitterStatistics;
+        private readonly IExtractionStatisticsHandler _extractionStatistics;
+
 
         #region Ctor
-        public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics)
+        public StatisticsController(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ITwitterStatisticsHandler twitterStatistics, IExtractionStatisticsHandler extractionStatistics)
         {
             _twitterUserDal = twitterUserDal;
             _followersDal = followersDal;
             _twitterStatistics = twitterStatistics;
+            _extractionStatistics = extractionStatistics;
         }
         #endregion
 
@@ -29,7 +33,8 @@ namespace BirdsiteLive.Controllers
             {
                 FollowersCount = await _followersDal.GetFollowersCountAsync(),
                 TwitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync(),
-                TwitterStatistics = _twitterStatistics.GetStatistics()
+                TwitterStatistics = _twitterStatistics.GetStatistics(),
+                ExtractionStatistics = _extractionStatistics.GetStatistics(),
             };
             return View(stats);
         }

src/BirdsiteLive/Models/StatisticsModels/Statistics.cs

@@ -1,4 +1,5 @@
-using BirdsiteLive.Twitter.Models;
+using BirdsiteLive.Domain.Statistics;
+using BirdsiteLive.Twitter.Models;
 
 namespace BirdsiteLive.Models.StatisticsModels
 {
@@ -7,5 +8,6 @@ namespace BirdsiteLive.Models.StatisticsModels
         public int FollowersCount { get; set; }
         public int TwitterUserCount { get; set; }
         public ApiStatistics TwitterStatistics { get; set; }
+        public ExtractionStatistics ExtractionStatistics { get; set; }
     }
 }

src/BirdsiteLive/Views/Statistics/Index.cshtml

@@ -12,9 +12,17 @@
     <li>Followers: @Model.FollowersCount</li>
 </ul>
 
-<h4>Twitter API</h4>
+<h4>Twitter API (Min, Avg, Max for the last 24h)</h4>
 <ul>
-    <li>Users Calls: @Model.TwitterStatistics.UserCallsCount / @Model.TwitterStatistics.UserCallsMax</li>
+    <li>Users Calls: @Model.TwitterStatistics.UserCallsCountMin ;  @Model.TwitterStatistics.UserCallsCountAvg ;  @Model.TwitterStatistics.UserCallsCountMax / @Model.TwitterStatistics.UserCallsMax</li>
-    <li>Tweets Calls: @Model.TwitterStatistics.TweetCallsCount / @Model.TwitterStatistics.TweetCallsMax</li>
+    <li>Tweets Calls: @Model.TwitterStatistics.TweetCallsCountMin ; @Model.TwitterStatistics.TweetCallsCountAvg ; @Model.TwitterStatistics.TweetCallsCountMax / @Model.TwitterStatistics.TweetCallsMax</li>
-    <li>Timeline Calls: @Model.TwitterStatistics.TimelineCallsCount / @Model.TwitterStatistics.TimelineCallsMax</li>
+    <li>Timeline Calls: @Model.TwitterStatistics.TimelineCallsCountMin ; @Model.TwitterStatistics.TimelineCallsCountAvg ; @Model.TwitterStatistics.TimelineCallsCountMax / @Model.TwitterStatistics.TimelineCallsMax</li>
+</ul>
+
+<h4>Mentions Extraction</h4>
+<ul>
+    <li>Current day in descriptions:  @Model.ExtractionStatistics.MentionsInDescriptionsExtraction</li>
+    <li>Current day in statuses: @Model.ExtractionStatistics.MentionsInStatusesExtraction</li>
+    <li>Last day in descriptions: @Model.ExtractionStatistics.LastMentionsInDescriptionsExtraction</li>
+    <li>Last day in statuses: @Model.ExtractionStatistics.LastMentionsInStatusesExtraction</li>
 </ul>