From 2463b53363a621c1e18223bda5c47a663707a22c Mon Sep 17 00:00:00 2001
From: Matt Jankowski <matt@jankowski.online>
Date: Thu, 21 Dec 2023 03:51:03 -0500
Subject: [PATCH] More duplicates in cli maintenance spec, misc bug fixes
(#28449)
---
lib/mastodon/cli/maintenance.rb | 40 +-
spec/lib/mastodon/cli/maintenance_spec.rb | 426 +++++++++++++++++++++-
2 files changed, 443 insertions(+), 23 deletions(-)
diff --git a/lib/mastodon/cli/maintenance.rb b/lib/mastodon/cli/maintenance.rb
index 98067c6e34..7b3a9852a6 100644
--- a/lib/mastodon/cli/maintenance.rb
+++ b/lib/mastodon/cli/maintenance.rb
@@ -40,6 +40,10 @@ module Mastodon::CLI
class BulkImport < ApplicationRecord; end
class SoftwareUpdate < ApplicationRecord; end
+ class DomainBlock < ApplicationRecord
+ scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) }
+ end
+
class PreviewCard < ApplicationRecord
self.inheritance_column = false
end
@@ -249,19 +253,7 @@ module Mastodon::CLI
say 'Deduplicating user records…'
- # Deduplicating email
- ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
- users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
- ref_user = users.shift
- say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
- say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
- say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
-
- users.each_with_index do |user, index|
- user.update!(email: "#{index} " + user.email)
- end
- end
-
+ deduplicate_users_process_email
deduplicate_users_process_confirmation_token
deduplicate_users_process_remember_token
deduplicate_users_process_password_token
@@ -280,6 +272,20 @@ module Mastodon::CLI
ActiveRecord::Base.connection.execute('REINDEX INDEX index_users_on_unconfirmed_email;') if migrator_version >= 2023_07_02_151753
end
+ def deduplicate_users_process_email
+ ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
+ users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
+ ref_user = users.shift
+ say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
+ say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
+ say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
+
+ users.each_with_index do |user, index|
+ user.update!(email: "#{index} " + user.email)
+ end
+ end
+ end
+
def deduplicate_users_process_confirmation_token
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
@@ -571,7 +577,7 @@ module Mastodon::CLI
say 'Deduplicating webhooks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row|
- Webhooks.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
+ Webhook.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
say 'Restoring webhooks indexes…'
@@ -604,11 +610,7 @@ module Mastodon::CLI
say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
- ref_id = ask('Account to keep unchanged:') do |q|
- q.required true
- q.default 0
- q.convert :int
- end
+ ref_id = ask('Account to keep unchanged:', required: true, default: 0).to_i
accounts.delete_at(ref_id)
diff --git a/spec/lib/mastodon/cli/maintenance_spec.rb b/spec/lib/mastodon/cli/maintenance_spec.rb
index 353bf08b68..ca492bbf69 100644
--- a/spec/lib/mastodon/cli/maintenance_spec.rb
+++ b/spec/lib/mastodon/cli/maintenance_spec.rb
@@ -62,6 +62,7 @@ describe Mastodon::CLI::Maintenance do
context 'with duplicate accounts' do
before do
prepare_duplicate_data
+ choose_local_account_to_keep
end
let(:duplicate_account_username) { 'username' }
@@ -71,21 +72,37 @@ describe Mastodon::CLI::Maintenance do
expect { subject }
.to output_results(
'Deduplicating accounts',
+ 'Multiple local accounts were found for',
'Restoring index_accounts_on_username_and_domain_lower',
'Reindexing textual indexes on accounts…',
'Finished!'
)
- .and change(duplicate_accounts, :count).from(2).to(1)
+ .and change(duplicate_remote_accounts, :count).from(2).to(1)
+ .and change(duplicate_local_accounts, :count).from(2).to(1)
end
- def duplicate_accounts
+ def duplicate_remote_accounts
Account.where(username: duplicate_account_username, domain: duplicate_account_domain)
end
+ def duplicate_local_accounts
+ Account.where(username: duplicate_account_username, domain: nil)
+ end
+
def prepare_duplicate_data
ActiveRecord::Base.connection.remove_index :accounts, name: :index_accounts_on_username_and_domain_lower
- Fabricate(:account, username: duplicate_account_username, domain: duplicate_account_domain)
- Fabricate.build(:account, username: duplicate_account_username, domain: duplicate_account_domain).save(validate: false)
+ _remote_account = Fabricate(:account, username: duplicate_account_username, domain: duplicate_account_domain)
+ _remote_account_dupe = Fabricate.build(:account, username: duplicate_account_username, domain: duplicate_account_domain).save(validate: false)
+ _local_account = Fabricate(:account, username: duplicate_account_username, domain: nil)
+ _local_account_dupe = Fabricate.build(:account, username: duplicate_account_username, domain: nil).save(validate: false)
+ end
+
+ def choose_local_account_to_keep
+ allow(cli.shell)
+ .to receive(:ask)
+ .with(/Account to keep unchanged/, anything)
+ .and_return('0')
+ .once
end
end
@@ -175,6 +192,407 @@ describe Mastodon::CLI::Maintenance do
end
end
+ context 'with duplicate account_domain_blocks' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:duplicate_domain) { 'example.host' }
+ let(:account) { Fabricate(:account) }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Removing duplicate account domain blocks',
+ 'Restoring account domain blocks indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_account_domain_blocks, :count).from(2).to(1)
+ end
+
+ def duplicate_account_domain_blocks
+ AccountDomainBlock.where(account: account, domain: duplicate_domain)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :account_domain_blocks, [:account_id, :domain]
+ Fabricate(:account_domain_block, account: account, domain: duplicate_domain)
+ Fabricate.build(:account_domain_block, account: account, domain: duplicate_domain).save(validate: false)
+ end
+ end
+
+ context 'with duplicate announcement_reactions' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:account) { Fabricate(:account) }
+ let(:announcement) { Fabricate(:announcement) }
+ let(:name) { Fabricate(:custom_emoji).shortcode }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Removing duplicate announcement reactions',
+ 'Restoring announcement_reactions indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_announcement_reactions, :count).from(2).to(1)
+ end
+
+ def duplicate_announcement_reactions
+ AnnouncementReaction.where(account: account, announcement: announcement, name: name)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :announcement_reactions, [:account_id, :announcement_id, :name]
+ Fabricate(:announcement_reaction, account: account, announcement: announcement, name: name)
+ Fabricate.build(:announcement_reaction, account: account, announcement: announcement, name: name).save(validate: false)
+ end
+ end
+
+ context 'with duplicate conversations' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:uri) { 'https://example.host/path' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating conversations',
+ 'Restoring conversations indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_conversations, :count).from(2).to(1)
+ end
+
+ def duplicate_conversations
+ Conversation.where(uri: uri)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :conversations, :uri
+ Fabricate(:conversation, uri: uri)
+ Fabricate.build(:conversation, uri: uri).save(validate: false)
+ end
+ end
+
+ context 'with duplicate custom_emojis' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:duplicate_shortcode) { 'wowzers' }
+ let(:duplicate_domain) { 'example.host' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating custom_emojis',
+ 'Restoring custom_emojis indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_custom_emojis, :count).from(2).to(1)
+ end
+
+ def duplicate_custom_emojis
+ CustomEmoji.where(shortcode: duplicate_shortcode, domain: duplicate_domain)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :custom_emojis, [:shortcode, :domain]
+ Fabricate(:custom_emoji, shortcode: duplicate_shortcode, domain: duplicate_domain)
+ Fabricate.build(:custom_emoji, shortcode: duplicate_shortcode, domain: duplicate_domain).save(validate: false)
+ end
+ end
+
+ context 'with duplicate custom_emoji_categories' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:duplicate_name) { 'name_value' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating custom_emoji_categories',
+ 'Restoring custom_emoji_categories indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_custom_emoji_categories, :count).from(2).to(1)
+ end
+
+ def duplicate_custom_emoji_categories
+ CustomEmojiCategory.where(name: duplicate_name)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :custom_emoji_categories, :name
+ Fabricate(:custom_emoji_category, name: duplicate_name)
+ Fabricate.build(:custom_emoji_category, name: duplicate_name).save(validate: false)
+ end
+ end
+
+ context 'with duplicate domain_allows' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:domain) { 'example.host' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating domain_allows',
+ 'Restoring domain_allows indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_domain_allows, :count).from(2).to(1)
+ end
+
+ def duplicate_domain_allows
+ DomainAllow.where(domain: domain)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :domain_allows, :domain
+ Fabricate(:domain_allow, domain: domain)
+ Fabricate.build(:domain_allow, domain: domain).save(validate: false)
+ end
+ end
+
+ context 'with duplicate domain_blocks' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:domain) { 'example.host' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating domain_blocks',
+ 'Restoring domain_blocks indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_domain_blocks, :count).from(2).to(1)
+ end
+
+ def duplicate_domain_blocks
+ DomainBlock.where(domain: domain)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :domain_blocks, :domain
+ Fabricate(:domain_block, domain: domain)
+ Fabricate.build(:domain_block, domain: domain).save(validate: false)
+ end
+ end
+
+ context 'with duplicate email_domain_blocks' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:domain) { 'example.host' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating email_domain_blocks',
+ 'Restoring email_domain_blocks indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_email_domain_blocks, :count).from(2).to(1)
+ end
+
+ def duplicate_email_domain_blocks
+ EmailDomainBlock.where(domain: domain)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :email_domain_blocks, :domain
+ Fabricate(:email_domain_block, domain: domain)
+ Fabricate.build(:email_domain_block, domain: domain).save(validate: false)
+ end
+ end
+
+ context 'with duplicate media_attachments' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:shortcode) { 'codenam' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating media_attachments',
+ 'Restoring media_attachments indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_media_attachments, :count).from(2).to(1)
+ end
+
+ def duplicate_media_attachments
+ MediaAttachment.where(shortcode: shortcode)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :media_attachments, :shortcode
+ Fabricate(:media_attachment, shortcode: shortcode)
+ Fabricate.build(:media_attachment, shortcode: shortcode).save(validate: false)
+ end
+ end
+
+ context 'with duplicate preview_cards' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:url) { 'https://example.host/path' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating preview_cards',
+ 'Restoring preview_cards indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_preview_cards, :count).from(2).to(1)
+ end
+
+ def duplicate_preview_cards
+ PreviewCard.where(url: url)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :preview_cards, :url
+ Fabricate(:preview_card, url: url)
+ Fabricate.build(:preview_card, url: url).save(validate: false)
+ end
+ end
+
+ context 'with duplicate statuses' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:uri) { 'https://example.host/path' }
+ let(:account) { Fabricate(:account) }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating statuses',
+ 'Restoring statuses indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_statuses, :count).from(2).to(1)
+ end
+
+ def duplicate_statuses
+ Status.where(uri: uri)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :statuses, :uri
+ Fabricate(:status, account: account, uri: uri)
+ duplicate = Fabricate.build(:status, account: account, uri: uri)
+ duplicate.save(validate: false)
+ Fabricate(:status_pin, account: account, status: duplicate)
+ Fabricate(:status, in_reply_to_id: duplicate.id)
+ Fabricate(:status, reblog_of_id: duplicate.id)
+ end
+ end
+
+ context 'with duplicate tags' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:name) { 'tagname' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating tags',
+ 'Restoring tags indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_tags, :count).from(2).to(1)
+ end
+
+ def duplicate_tags
+ Tag.where(name: name)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :tags, name: 'index_tags_on_name_lower_btree'
+ Fabricate(:tag, name: name)
+ Fabricate.build(:tag, name: name).save(validate: false)
+ end
+ end
+
+ context 'with duplicate webauthn_credentials' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:external_id) { '123_123_123' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating webauthn_credentials',
+ 'Restoring webauthn_credentials indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_webauthn_credentials, :count).from(2).to(1)
+ end
+
+ def duplicate_webauthn_credentials
+ WebauthnCredential.where(external_id: external_id)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :webauthn_credentials, :external_id
+ Fabricate(:webauthn_credential, external_id: external_id)
+ Fabricate.build(:webauthn_credential, external_id: external_id).save(validate: false)
+ end
+ end
+
+ context 'with duplicate webhooks' do
+ before do
+ prepare_duplicate_data
+ end
+
+ let(:url) { 'https://example.host/path' }
+
+ it 'runs the deduplication process' do
+ expect { subject }
+ .to output_results(
+ 'Deduplicating webhooks',
+ 'Restoring webhooks indexes',
+ 'Finished!'
+ )
+ .and change(duplicate_webhooks, :count).from(2).to(1)
+ end
+
+ def duplicate_webhooks
+ Webhook.where(url: url)
+ end
+
+ def prepare_duplicate_data
+ ActiveRecord::Base.connection.remove_index :webhooks, :url
+ Fabricate(:webhook, url: url)
+ Fabricate.build(:webhook, url: url).save(validate: false)
+ end
+ end
+
def agree_to_backup_warning
allow(cli.shell)
.to receive(:yes?)