From 7c2dcdbcece39b2e552fce971cf21b21f415976b Mon Sep 17 00:00:00 2001 From: Vincent Cloutier Date: Fri, 2 Jun 2023 14:12:27 -0400 Subject: [PATCH] progress on wikidata sync --- .../BirdsiteLive.Wikidata.csproj | 5 +++ src/BirdsiteLive.Wikidata/Program.cs | 27 +++++++++++++++- src/BirdsiteLive.Wikidata/README.md | 31 +++++++++++++++++++ .../TwitterUserPostgresDal.cs | 13 ++++++++ .../Models/SyncTwitterUser.cs | 1 + 5 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/BirdsiteLive.Wikidata/BirdsiteLive.Wikidata.csproj b/src/BirdsiteLive.Wikidata/BirdsiteLive.Wikidata.csproj index 2b14c81..49c83ca 100644 --- a/src/BirdsiteLive.Wikidata/BirdsiteLive.Wikidata.csproj +++ b/src/BirdsiteLive.Wikidata/BirdsiteLive.Wikidata.csproj @@ -7,4 +7,9 @@ enable + + + + + diff --git a/src/BirdsiteLive.Wikidata/Program.cs b/src/BirdsiteLive.Wikidata/Program.cs index 1b861e4..9813148 100644 --- a/src/BirdsiteLive.Wikidata/Program.cs +++ b/src/BirdsiteLive.Wikidata/Program.cs @@ -3,6 +3,24 @@ using System.IO; using System.Net.Http; using System.Threading.Tasks; using System.Xml; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.DAL.Postgres.DataAccessLayers; +using BirdsiteLive.DAL.Postgres.Settings; + +var settings = new PostgresSettings() +{ + ConnString = System.Environment.GetEnvironmentVariable("ConnString"), +}; +var dal = new TwitterUserPostgresDal(settings); + +var twitterUser = new HashSet(); +var twitterUserQuery = await dal.GetAllTwitterUsersAsync(); +Console.WriteLine("Loading twitter users"); +foreach (SyncTwitterUser user in twitterUserQuery) +{ + twitterUser.Add(user.Acct); +} +Console.WriteLine("Done loading twitter users"); Console.WriteLine("Hello, World!"); var client = new HttpClient(); @@ -18,6 +36,13 @@ var content = await response.Content.ReadAsStringAsync(); foreach (string n in content.Split("\n")) { var s = n.Split(","); - Console.WriteLine(s[0]); + if (n.Length < 2) + continue; + + var acct = s[1]; + var fedi = s[2]; + await dal.UpdateTwitterUserFediAcctAsync(acct, fedi); + if (twitterUser.Contains(acct)) + Console.WriteLine(fedi); } diff --git a/src/BirdsiteLive.Wikidata/README.md b/src/BirdsiteLive.Wikidata/README.md index fd05dfd..88bfa88 100644 --- a/src/BirdsiteLive.Wikidata/README.md +++ b/src/BirdsiteLive.Wikidata/README.md @@ -1,2 +1,33 @@ # Wikidata service +Wikidata is the metadata community behind Wikipedia. See for example +[Hank Green](https://www.wikidata.org/wiki/Q550996). In his page, there are +all the links to his wikipedia pages, and many facts about him. What is +particularly useful to us is the twitter username (P2002) and mastodon +username (P4033). + +From this information, we can build a feature that suggests to follow the +native fediverse account of someone you are trying to follow from Twitter. + +The main downside is that those redirect are only for somewhat famous +people/organisations. + +## Goals +### Being reusable by others +All this data can be useful to many other fediverse projects: tools +for finding interesting accounts to follow, "verified" badge powered by +Wikipedia, etc. I hope that by working on improving this dataset, we can +help other projects thrive. +### Being independent of Twitter +Bird.makeup has to build features in a way that can't be suddenly cut off. +Building this feature with a "Log in with Twitter" is not viable. +Wikipedia is independent and outside of Elon's reach. + +Also this system supports many other services: TikTok, Reddit, YouTube, etc. +Which is really useful to expend the scope of this project while reusing as +much work as possible +### Having great moderation + +Wikipedia has many tools to help curate data and remove troll's submissions, +far better than anything I can build. I much prefer contribute to what +they are doing than try to compete \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 5fe3b6c..1681c7c 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -172,6 +172,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers } } + public async Task UpdateTwitterUserFediAcctAsync(string twitterUsername, string fediUsername) + { + if(twitterUsername == default) throw new ArgumentException("id"); + + var query = $"UPDATE {_settings.TwitterUserTableName} SET fediverseaccount = $1 WHERE acct = $2"; + await using var connection = DataSource.CreateConnection(); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand(query, connection) { + Parameters = { new() { Value = fediUsername}, new() { Value = twitterUsername}} + }; + + await command.ExecuteNonQueryAsync(); + } public async Task UpdateTwitterUserIdAsync(string username, long twitterUserId) { if(username == default) throw new ArgumentException("id"); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index d7a8f24..4cace09 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -7,6 +7,7 @@ namespace BirdsiteLive.DAL.Models public int Id { get; set; } public long TwitterUserId { get; set; } public string Acct { get; set; } + public string FediAcct { get; set; } public long LastTweetPostedId { get; set; }