extract only needed info from tweets

src/BirdsiteLive.Domain/StatusService.cs

@@ -1,8 +1,10 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Models;
 using Tweetinvi.Models.Entities;
 
@@ -10,7 +12,7 @@ namespace BirdsiteLive.Domain
 {
     public interface IStatusService
     {
-        Note GetStatus(string username, ITweet tweet);
+        Note GetStatus(string username, ExtractedTweet tweet);
     }
 
     public class StatusService : IStatusService
@@ -24,7 +26,7 @@ namespace BirdsiteLive.Domain
         }
         #endregion
 
-        public Note GetStatus(string username, ITweet tweet)
+        public Note GetStatus(string username, ExtractedTweet tweet)
         {
             var actorUrl = $"https://{_instanceSettings.Domain}/users/{username}";
             var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{tweet.Id}";
@@ -49,8 +51,8 @@ namespace BirdsiteLive.Domain
                 //cc = new string[0],
 
                 sensitive = false,
-                content = $"<p>{tweet.Text}</p>",
+                content = $"<p>{tweet.MessageContent}</p>",
-                attachment = GetAttachments(tweet.Media),
+                attachment = Convert(tweet.Media),
                 tag = new string[0]
             };
           
@@ -58,62 +60,17 @@ namespace BirdsiteLive.Domain
             return note;
         }
 
-        private Attachment[] GetAttachments(List<IMediaEntity> media)
+        private Attachment[] Convert(ExtractedMedia[] media)
         {
-            var result = new List<Attachment>();
+            return media.Select(x =>
-
-            foreach (var m in media)
             {
-                var mediaUrl = GetMediaUrl(m);
+                return new Attachment
-                var mediaType = GetMediaType(m.MediaType, mediaUrl);
-                if (mediaType == null) continue;
-
-                var att = new Attachment
                 {
                     type = "Document",
-                    mediaType = mediaType,
+                    url = x.Url,
-                    url = mediaUrl
+                    mediaType = x.MediaType
                 };
-                result.Add(att);
+            }).ToArray();
-            }
-
-            return result.ToArray();
-        }
-
-        private string GetMediaUrl(IMediaEntity media)
-        {
-            switch (media.MediaType)
-            {
-                case "photo": return media.MediaURLHttps;
-                case "animated_gif": return media.VideoDetails.Variants[0].URL;
-                case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL;
-                default: return null;
-            }
-        }
-
-        private string GetMediaType(string mediaType, string mediaUrl)
-        {
-            switch (mediaType)
-            {
-                case "photo":
-                    var ext = Path.GetExtension(mediaUrl);
-                    switch (ext)
-                    {
-                        case ".jpg":
-                        case ".jpeg":
-                            return "image/jpeg";
-                        case ".png":
-                            return "image/png";
-                    }
-                    return null;
-
-                case "animated_gif":
-                    return "image/gif";
-
-                case "video":
-                    return "video/mp4";
-            }
-            return null;
         }
     }
 }

src/BirdsiteLive.Pipeline/Models/UserWithTweetsToSync.cs

@@ -1,4 +1,5 @@
 using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Models
@@ -6,7 +7,7 @@ namespace BirdsiteLive.Pipeline.Models
     public class UserWithTweetsToSync
     {
         public SyncTwitterUser User { get; set; }
-        public ITweet[] Tweets { get; set; }
+        public ExtractedTweet[] Tweets { get; set; }
         public Follower[] Followers { get; set; }
     }
 }

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -7,6 +7,7 @@ using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
+using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Processors
@@ -51,9 +52,9 @@ namespace BirdsiteLive.Pipeline.Processors
             return usersWtTweets.ToArray();
         }
 
-        private ITweet[] RetrieveNewTweets(SyncTwitterUser user)
+        private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
         {
-            ITweet[] tweets;
+            ExtractedTweet[] tweets;
             if (user.LastTweetPostedId == -1)
                 tweets = _twitterService.GetTimeline(user.Acct, 1);
             else

src/BirdsiteLive.Pipeline/Processors/SendTweetsToFollowersProcessor.cs

@@ -10,6 +10,7 @@ using BirdsiteLive.Domain;
 using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
+using BirdsiteLive.Twitter.Models;
 using Tweetinvi.Models;
 
 namespace BirdsiteLive.Pipeline.Processors
@@ -50,7 +51,7 @@ namespace BirdsiteLive.Pipeline.Processors
             return userWithTweetsToSync;
         }
 
-        private async Task ProcessFollowerAsync(IEnumerable<ITweet> tweets, Follower follower, int userId,
+        private async Task ProcessFollowerAsync(IEnumerable<ExtractedTweet> tweets, Follower follower, int userId,
             SyncTwitterUser user)
         {
             var fromStatusId = follower.FollowingsSyncStatus[userId];

src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs

@@ -0,0 +1,8 @@
+namespace BirdsiteLive.Twitter.Models
+{
+    public class ExtractedMedia
+    {
+        public string MediaType { get; set; }
+        public string Url { get; set; }
+    }
+}

src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Net.Sockets;
+
+namespace BirdsiteLive.Twitter.Models
+{
+    public class ExtractedTweet
+    {
+        public long Id { get; set; }
+        public long? InReplyToStatusId { get; set; }
+        public string MessageContent { get; set; }
+        public ExtractedMedia[] Media { get; set; }
+        public DateTime CreatedAt { get; set; }
+    }
+}

src/BirdsiteLive.Twitter/TwitterService.cs

@@ -1,11 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Twitter.Models;
 using Tweetinvi;
 using Tweetinvi.Models;
+using Tweetinvi.Models.Entities;
 using Tweetinvi.Parameters;
 
 namespace BirdsiteLive.Twitter
@@ -13,8 +15,8 @@ namespace BirdsiteLive.Twitter
     public interface ITwitterService
     {
         TwitterUser GetUser(string username);
-        ITweet GetTweet(long statusId);
+        ExtractedTweet GetTweet(long statusId);
-        ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1);
+        ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1);
     }
 
     public class TwitterService : ITwitterService
@@ -31,7 +33,6 @@ namespace BirdsiteLive.Twitter
 
         public TwitterUser GetUser(string username)
         {
-            //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true);
             var user = User.GetUserFromScreenName(username);
             if (user == null) return null;
 
@@ -47,16 +48,104 @@ namespace BirdsiteLive.Twitter
             };
         }
 
-        public ITweet GetTweet(long statusId)
+        public ExtractedTweet GetTweet(long statusId)
         {
-            //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true);
+            TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
             var tweet = Tweet.GetTweet(statusId);
-            return tweet;
+            return Extract(tweet);
         }
 
-        public ITweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
+        private ExtractedTweet Extract(ITweet tweet)
+        {
+            var extractedTweet = new ExtractedTweet
+            {
+                Id = tweet.Id,
+                InReplyToStatusId = tweet.InReplyToStatusId,
+                MessageContent = ExtractMessage(tweet),
+                Media = ExtractMedia(tweet.Media),
+                CreatedAt = tweet.CreatedAt
+            };
+            return extractedTweet;
+        }
+        
+        private string ExtractMessage(ITweet tweet)
+        {
+            var tweetUrls = tweet.Media.Select(x => x.URL).Distinct();
+            var message = tweet.FullText;
+            foreach (var tweetUrl in tweetUrls)
+                message = message.Replace(tweetUrl, string.Empty).Trim();
+
+            if (tweet.QuotedTweet != null) message = $"[Quote RT] {message}";
+            if (tweet.IsRetweet)
+            {
+                if (tweet.RetweetedTweet != null)
+                    message = $"[RT {tweet.RetweetedTweet.CreatedBy.ScreenName}] {tweet.RetweetedTweet.FullText}";
+                else
+                    message = message.Replace("RT", "[RT]");
+            }
+
+            return message;
+        }
+        
+        private ExtractedMedia[] ExtractMedia(List<IMediaEntity> media)
+        {
+            var result = new List<ExtractedMedia>();
+
+            foreach (var m in media)
+            {
+                var mediaUrl = GetMediaUrl(m);
+                var mediaType = GetMediaType(m.MediaType, mediaUrl);
+                if (mediaType == null) continue;
+
+                var att = new ExtractedMedia
+                {
+                    MediaType = mediaType,
+                    Url = mediaUrl
+                };
+                result.Add(att);
+            }
+
+            return result.ToArray();
+        }
+
+        private string GetMediaUrl(IMediaEntity media)
+        {
+            switch (media.MediaType)
+            {
+                case "photo": return media.MediaURLHttps;
+                case "animated_gif": return media.VideoDetails.Variants[0].URL;
+                case "video": return media.VideoDetails.Variants.OrderByDescending(x => x.Bitrate).First().URL;
+                default: return null;
+            }
+        }
+
+        private string GetMediaType(string mediaType, string mediaUrl)
+        {
+            switch (mediaType)
+            {
+                case "photo":
+                    var ext = Path.GetExtension(mediaUrl);
+                    switch (ext)
+                    {
+                        case ".jpg":
+                        case ".jpeg":
+                            return "image/jpeg";
+                        case ".png":
+                            return "image/png";
+                    }
+                    return null;
+
+                case "animated_gif":
+                    return "image/gif";
+
+                case "video":
+                    return "video/mp4";
+            }
+            return null;
+        }
+
+        public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)
         {
-            //Auth.SetApplicationOnlyCredentials(_settings.ConsumerKey, _settings.ConsumerSecret, true);
             TweetinviConfig.CurrentThreadSettings.TweetMode = TweetMode.Extended;
 
             var user = User.GetUserFromScreenName(username);
@@ -77,7 +166,7 @@ namespace BirdsiteLive.Twitter
                 if (timeline != null) tweets.AddRange(timeline);
             }
 
-            return tweets.ToArray();
+            return tweets.Select(Extract).ToArray();
             //return tweets.Where(x => returnReplies || string.IsNullOrWhiteSpace(x.InReplyToScreenName)).ToArray();
         }
     }