583 lines
20 KiB
Ruby
583 lines
20 KiB
Ruby
namespace :After do
|
||
# Keep only the most recent tasks, i.e., about two years' worth.
|
||
# If you need older tasks, check GitHub.
|
||
|
||
desc "Update the mapping for the work index"
|
||
task(update_work_mapping: :environment) do
|
||
WorkIndexer.create_mapping
|
||
end
|
||
|
||
desc "Fix tags with extra spaces"
|
||
task(fix_tags_with_extra_spaces: :environment) do
|
||
total_tags = Tag.count
|
||
total_batches = (total_tags + 999) / 1000
|
||
puts "Inspecting #{total_tags} tags in #{total_batches} batches"
|
||
|
||
report_string = ["Tag ID", "Old tag name", "New tag name"].to_csv
|
||
Tag.find_in_batches.with_index do |batch, index|
|
||
batch_number = index + 1
|
||
progress_msg = "Batch #{batch_number} of #{total_batches} complete"
|
||
|
||
batch.each do |tag|
|
||
next unless tag.name != tag.name.squish
|
||
|
||
old_tag_name = tag.name
|
||
new_tag_name = old_tag_name.gsub(/[[:space:]]/, "_")
|
||
|
||
new_tag_name << "_" while Tag.find_by(name: new_tag_name)
|
||
tag.update_attribute(:name, new_tag_name)
|
||
|
||
report_row = [tag.id, old_tag_name, new_tag_name].to_csv
|
||
report_string += report_row
|
||
end
|
||
|
||
puts(progress_msg) && STDOUT.flush
|
||
end
|
||
puts(report_string) && STDOUT.flush
|
||
end
|
||
|
||
desc "Fix works imported with a noncanonical Teen & Up Audiences rating tag"
|
||
task(fix_teen_and_up_imported_rating: :environment) do
|
||
borked_rating_tag = Rating.find_by!(name: "Teen & Up Audiences")
|
||
canonical_rating_tag = Rating.find_by!(name: ArchiveConfig.RATING_TEEN_TAG_NAME)
|
||
|
||
work_ids = []
|
||
invalid_work_ids = []
|
||
borked_rating_tag.works.find_each do |work|
|
||
work.ratings << canonical_rating_tag
|
||
work.ratings = work.ratings - [borked_rating_tag]
|
||
if work.save
|
||
work_ids << work.id
|
||
else
|
||
invalid_work_ids << work.id
|
||
end
|
||
print(".") && STDOUT.flush
|
||
end
|
||
|
||
unless work_ids.empty?
|
||
puts "Converted '#{borked_rating_tag.name}' rating tag on #{work_ids.size} works:"
|
||
puts work_ids.join(", ")
|
||
STDOUT.flush
|
||
end
|
||
|
||
unless invalid_work_ids.empty?
|
||
puts "The following #{invalid_work_ids.size} works failed validations and could not be saved:"
|
||
puts invalid_work_ids.join(", ")
|
||
STDOUT.flush
|
||
end
|
||
end
|
||
|
||
desc "Clean up multiple rating tags"
|
||
task(clean_up_multiple_ratings: :environment) do
|
||
default_rating_tag = Rating.find_by!(name: ArchiveConfig.RATING_DEFAULT_TAG_NAME)
|
||
es_results = $elasticsearch.search(index: WorkIndexer.index_name, body: {
|
||
query: {
|
||
bool: {
|
||
filter: {
|
||
script: {
|
||
script: {
|
||
source: "doc['rating_ids'].length > 1",
|
||
lang: "painless"
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
})
|
||
invalid_works = QueryResult.new("Work", es_results)
|
||
|
||
puts "There are #{invalid_works.size} works with multiple ratings."
|
||
|
||
fixed_work_ids = []
|
||
unfixed_word_ids = []
|
||
invalid_works.each do |work|
|
||
work.ratings = [default_rating_tag]
|
||
work.rating_string = default_rating_tag.name
|
||
|
||
if work.save
|
||
fixed_work_ids << work.id
|
||
else
|
||
unfixed_word_ids << work.id
|
||
end
|
||
print(".") && $stdout.flush
|
||
end
|
||
|
||
unless fixed_work_ids.empty?
|
||
puts "Cleaned up having multiple ratings on #{fixed_work_ids.size} works:"
|
||
puts fixed_work_ids.join(", ")
|
||
$stdout.flush
|
||
end
|
||
|
||
unless unfixed_word_ids.empty?
|
||
puts "The following #{unfixed_word_ids.size} works failed validations and could not be saved:"
|
||
puts unfixed_word_ids.join(", ")
|
||
$stdout.flush
|
||
end
|
||
end
|
||
|
||
desc "Clean up noncanonical rating tags"
|
||
task(clean_up_noncanonical_ratings: :environment) do
|
||
canonical_not_rated_tag = Rating.find_by!(name: ArchiveConfig.RATING_DEFAULT_TAG_NAME)
|
||
noncanonical_ratings = Rating.where(canonical: false)
|
||
puts "There are #{noncanonical_ratings.size} noncanonical rating tags."
|
||
|
||
next if noncanonical_ratings.empty?
|
||
|
||
puts "The following noncanonical Ratings will be changed into Additional Tags:"
|
||
puts noncanonical_ratings.map(&:name).join("\n")
|
||
|
||
work_ids = []
|
||
invalid_work_ids = []
|
||
noncanonical_ratings.find_each do |tag|
|
||
works_using_tag = tag.works
|
||
tag.update_attribute(:type, "Freeform")
|
||
|
||
works_using_tag.find_each do |work|
|
||
next unless work.ratings.empty?
|
||
|
||
work.ratings = [canonical_not_rated_tag]
|
||
if work.save
|
||
work_ids << work.id
|
||
else
|
||
invalid_work_ids << work.id
|
||
end
|
||
print(".") && STDOUT.flush
|
||
end
|
||
end
|
||
|
||
unless work_ids.empty?
|
||
puts "The following #{work_ids.size} works were left without a rating and successfully received the Not Rated rating:"
|
||
puts work_ids.join(", ")
|
||
STDOUT.flush
|
||
end
|
||
|
||
unless invalid_work_ids.empty?
|
||
puts "The following #{invalid_work_ids.size} works failed validations and could not be saved:"
|
||
puts invalid_work_ids.join(", ")
|
||
STDOUT.flush
|
||
end
|
||
end
|
||
|
||
desc "Clean up noncanonical category tags"
|
||
task(clean_up_noncanonical_categories: :environment) do
|
||
Category.where(canonical: false).find_each do |tag|
|
||
tag.update_attribute(:type, "Freeform")
|
||
puts "Noncanonical Category tag '#{tag.name}' was changed into an Additional Tag."
|
||
end
|
||
STDOUT.flush
|
||
end
|
||
|
||
desc "Add default rating to works missing a rating"
|
||
task(add_default_rating_to_works: :environment) do
|
||
work_count = Work.count
|
||
total_batches = (work_count + 999) / 1000
|
||
puts("Checking #{work_count} works in #{total_batches} batches") && STDOUT.flush
|
||
updated_works = []
|
||
|
||
Work.find_in_batches.with_index do |batch, index|
|
||
batch_number = index + 1
|
||
|
||
batch.each do |work|
|
||
next unless work.ratings.empty?
|
||
|
||
work.ratings << Rating.find_by!(name: ArchiveConfig.RATING_DEFAULT_TAG_NAME)
|
||
work.save
|
||
updated_works << work.id
|
||
end
|
||
puts("Batch #{batch_number} of #{total_batches} complete") && STDOUT.flush
|
||
end
|
||
puts("Added default rating to works: #{updated_works}") && STDOUT.flush
|
||
end
|
||
|
||
desc "Backfill renamed_at for existing users"
|
||
task(add_renamed_at_from_log: :environment) do
|
||
total_users = User.all.size
|
||
total_batches = (total_users + 999) / 1000
|
||
puts "Updating #{total_users} users in #{total_batches} batches"
|
||
|
||
User.find_in_batches.with_index do |batch, index|
|
||
batch.each do |user|
|
||
renamed_at_from_log = user.log_items.where(action: ArchiveConfig.ACTION_RENAME).last&.created_at
|
||
next unless renamed_at_from_log
|
||
|
||
user.update_column(:renamed_at, renamed_at_from_log)
|
||
end
|
||
|
||
batch_number = index + 1
|
||
progress_msg = "Batch #{batch_number} of #{total_batches} complete"
|
||
puts(progress_msg) && STDOUT.flush
|
||
end
|
||
puts && STDOUT.flush
|
||
end
|
||
|
||
desc "Fix threads for comments from 2009"
|
||
task(fix_2009_comment_threads: :environment) do
|
||
def fix_comment(comment)
|
||
comment.with_lock do
|
||
if comment.reply_comment?
|
||
comment.update_column(:thread, comment.commentable.thread)
|
||
else
|
||
comment.update_column(:thread, comment.id)
|
||
end
|
||
comment.comments.each { |reply| fix_comment(reply) }
|
||
end
|
||
end
|
||
|
||
incorrect = Comment.top_level.where("thread != id")
|
||
total = incorrect.count
|
||
|
||
puts "Updating #{total} thread(s)"
|
||
|
||
incorrect.find_each.with_index do |comment, index|
|
||
fix_comment(comment)
|
||
|
||
puts "Fixed thread #{index + 1} out of #{total}" if index % 100 == 99
|
||
end
|
||
end
|
||
|
||
desc "Remove translation_admin role"
|
||
task(remove_translation_admin_role: :environment) do
|
||
r = Role.find_by(name: "translation_admin")
|
||
r&.destroy
|
||
end
|
||
|
||
desc "Remove full-width and ideographic commas from tags"
|
||
task(remove_invalid_commas_from_tags: :environment) do
|
||
puts("Tags can only be renamed by an admin, who will be listed as the tag's last wrangler. Enter the admin login we should use:")
|
||
login = $stdin.gets.chomp.strip
|
||
admin = Admin.find_by(login: login)
|
||
|
||
if admin.present?
|
||
User.current_user = admin
|
||
|
||
["，", "、"].each do |comma|
|
||
tags = Tag.where("name LIKE ?", "%#{comma}%")
|
||
tags.each do |tag|
|
||
new_name = tag.name.gsub(/#{comma}/, "")
|
||
if tag.update(name: new_name) || tag.update(name: "#{new_name} - AO3-6626")
|
||
puts(tag.reload.name)
|
||
else
|
||
puts("Could not rename #{tag.reload.name}")
|
||
end
|
||
$stdout.flush
|
||
end
|
||
end
|
||
else
|
||
puts("Admin not found.")
|
||
end
|
||
end
|
||
|
||
desc "Add suffix to existing Underage Sex tag in preparation for Underage warning rename"
|
||
task(add_suffix_to_underage_sex_tag: :environment) do
|
||
puts("Tags can only be renamed by an admin, who will be listed as the tag's last wrangler. Enter the admin login we should use:")
|
||
login = $stdin.gets.chomp.strip
|
||
admin = Admin.find_by(login: login)
|
||
|
||
if admin.present?
|
||
User.current_user = admin
|
||
|
||
tag = Tag.find_by_name("Underage Sex")
|
||
|
||
if tag.blank?
|
||
puts("No Underage Sex tag found.")
|
||
elsif tag.is_a?(ArchiveWarning)
|
||
puts("Underage Sex is already an Archive Warning.")
|
||
else
|
||
suffixed_name = "Underage Sex - #{tag.class}"
|
||
if tag.update(name: suffixed_name)
|
||
puts("Renamed Underage Sex tag to #{tag.reload.name}.")
|
||
else
|
||
puts("Failed to rename Underage Sex tag to #{suffixed_name}.")
|
||
end
|
||
$stdout.flush
|
||
end
|
||
else
|
||
puts("Admin not found.")
|
||
end
|
||
end
|
||
|
||
desc "Rename Underage warning to Underage Sex"
|
||
task(rename_underage_warning: :environment) do
|
||
puts("Tags can only be renamed by an admin, who will be listed as the tag's last wrangler. Enter the admin login we should use:")
|
||
login = $stdin.gets.chomp.strip
|
||
admin = Admin.find_by(login: login)
|
||
|
||
if admin.present?
|
||
User.current_user = admin
|
||
|
||
tag = ArchiveWarning.find_by_name("Underage")
|
||
|
||
if tag.blank?
|
||
puts("No Underage warning tag found.")
|
||
else
|
||
new_name = "Underage Sex"
|
||
if tag.update(name: new_name)
|
||
puts("Renamed Underage warning tag to #{tag.reload.name}.")
|
||
else
|
||
puts("Failed to rename Underage warning tag to #{new_name}.")
|
||
end
|
||
$stdout.flush
|
||
end
|
||
else
|
||
puts("Admin not found.")
|
||
end
|
||
end
|
||
|
||
desc "Migrate collection icons to ActiveStorage paths"
|
||
task(migrate_collection_icons: :environment) do
|
||
require "aws-sdk-s3"
|
||
require "open-uri"
|
||
|
||
return unless Rails.env.staging? || Rails.env.production?
|
||
|
||
bucket_name = ENV["S3_BUCKET"]
|
||
prefix = "collections/icons/"
|
||
s3 = Aws::S3::Resource.new(
|
||
region: ENV["S3_REGION"],
|
||
access_key_id: ENV["S3_ACCESS_KEY_ID"],
|
||
secret_access_key: ENV["S3_SECRET_ACCESS_KEY"]
|
||
)
|
||
old_bucket = s3.bucket(bucket_name)
|
||
new_bucket = s3.bucket(ENV["TARGET_BUCKET"])
|
||
|
||
Collection.no_touching do
|
||
old_bucket.objects(prefix: prefix).each do |object|
|
||
# Path example: staging/icons/108621/original.png
|
||
path_parts = object.key.split("/")
|
||
next unless path_parts[-1]&.include?("original")
|
||
next if ActiveStorage::Attachment.where(record_type: "Collection", record_id: path_parts[-2]).any?
|
||
|
||
collection_id = path_parts[-2]
|
||
old_icon = URI.open("https://s3.amazonaws.com/#{bucket_name}/#{object.key}")
|
||
checksum = OpenSSL::Digest.new("MD5").tap do |result|
|
||
while (chunk = old_icon.read(5.megabytes))
|
||
result << chunk
|
||
end
|
||
old_icon.rewind
|
||
end.base64digest
|
||
|
||
key = nil
|
||
ActiveRecord::Base.transaction do
|
||
blob = ActiveStorage::Blob.create_before_direct_upload!(
|
||
filename: path_parts[-1],
|
||
byte_size: old_icon.size,
|
||
checksum: checksum,
|
||
content_type: Marcel::MimeType.for(old_icon)
|
||
)
|
||
key = blob.key
|
||
blob.attachments.create(
|
||
name: "icon",
|
||
record_type: "Collection",
|
||
record_id: collection_id
|
||
)
|
||
end
|
||
|
||
new_bucket.put_object(key: key, body: old_icon, acl: "bucket-owner-full-control")
|
||
puts "Finished collection #{collection_id}"
|
||
$stdout.flush
|
||
end
|
||
end
|
||
end
|
||
|
||
desc "Migrate pseud icons to ActiveStorage paths"
|
||
task(migrate_pseud_icons: :environment) do
|
||
require "aws-sdk-s3"
|
||
require "open-uri"
|
||
|
||
return unless Rails.env.staging? || Rails.env.production?
|
||
|
||
bucket_name = ENV["S3_BUCKET"]
|
||
prefix = Rails.env.production? ? "icons/" : "staging/icons/"
|
||
s3 = Aws::S3::Resource.new(
|
||
region: ENV["S3_REGION"],
|
||
access_key_id: ENV["S3_ACCESS_KEY_ID"],
|
||
secret_access_key: ENV["S3_SECRET_ACCESS_KEY"]
|
||
)
|
||
old_bucket = s3.bucket(bucket_name)
|
||
new_bucket = s3.bucket(ENV["TARGET_BUCKET"])
|
||
|
||
Pseud.no_touching do
|
||
old_bucket.objects(prefix: prefix).each do |object|
|
||
# Path example: staging/icons/108621/original.png
|
||
path_parts = object.key.split("/")
|
||
next unless path_parts[-1]&.include?("original")
|
||
next if ActiveStorage::Attachment.where(record_type: "Pseud", record_id: path_parts[-2]).any?
|
||
|
||
pseud_id = path_parts[-2]
|
||
old_icon = URI.open("https://s3.amazonaws.com/#{bucket_name}/#{object.key}")
|
||
checksum = OpenSSL::Digest.new("MD5").tap do |result|
|
||
while (chunk = old_icon.read(5.megabytes))
|
||
result << chunk
|
||
end
|
||
old_icon.rewind
|
||
end.base64digest
|
||
|
||
key = nil
|
||
ActiveRecord::Base.transaction do
|
||
blob = ActiveStorage::Blob.create_before_direct_upload!(
|
||
filename: path_parts[-1],
|
||
byte_size: old_icon.size,
|
||
checksum: checksum,
|
||
content_type: Marcel::MimeType.for(old_icon)
|
||
)
|
||
key = blob.key
|
||
blob.attachments.create(
|
||
name: "icon",
|
||
record_type: "Pseud",
|
||
record_id: pseud_id
|
||
)
|
||
end
|
||
|
||
new_bucket.put_object(key: key, body: old_icon, acl: "bucket-owner-full-control")
|
||
puts "Finished pseud #{pseud_id}"
|
||
$stdout.flush
|
||
end
|
||
end
|
||
end
|
||
|
||
desc "Migrate skin icons to ActiveStorage paths"
|
||
task(migrate_skin_icons: :environment) do
|
||
require "aws-sdk-s3"
|
||
require "open-uri"
|
||
|
||
return unless Rails.env.staging? || Rails.env.production?
|
||
|
||
bucket_name = ENV["S3_BUCKET"]
|
||
prefix = "skins/icons/"
|
||
s3 = Aws::S3::Resource.new(
|
||
region: ENV["S3_REGION"],
|
||
access_key_id: ENV["S3_ACCESS_KEY_ID"],
|
||
secret_access_key: ENV["S3_SECRET_ACCESS_KEY"]
|
||
)
|
||
old_bucket = s3.bucket(bucket_name)
|
||
new_bucket = s3.bucket(ENV["TARGET_BUCKET"])
|
||
|
||
Skin.no_touching do
|
||
old_bucket.objects(prefix: prefix).each do |object|
|
||
# Path example: staging/icons/108621/original.png
|
||
path_parts = object.key.split("/")
|
||
next unless path_parts[-1]&.include?("original")
|
||
next if ActiveStorage::Attachment.where(record_type: "Skin", record_id: path_parts[-2]).any?
|
||
|
||
skin_id = path_parts[-2]
|
||
old_icon = URI.open("https://s3.amazonaws.com/#{bucket_name}/#{object.key}")
|
||
checksum = OpenSSL::Digest.new("MD5").tap do |result|
|
||
while (chunk = old_icon.read(5.megabytes))
|
||
result << chunk
|
||
end
|
||
old_icon.rewind
|
||
end.base64digest
|
||
|
||
key = nil
|
||
ActiveRecord::Base.transaction do
|
||
blob = ActiveStorage::Blob.create_before_direct_upload!(
|
||
filename: path_parts[-1],
|
||
byte_size: old_icon.size,
|
||
checksum: checksum,
|
||
content_type: Marcel::MimeType.for(old_icon)
|
||
)
|
||
key = blob.key
|
||
blob.attachments.create(
|
||
name: "icon",
|
||
record_type: "Skin",
|
||
record_id: skin_id
|
||
)
|
||
end
|
||
|
||
new_bucket.put_object(key: key, body: old_icon, acl: "bucket-owner-full-control")
|
||
puts "Finished skin #{skin_id}"
|
||
$stdout.flush
|
||
end
|
||
end
|
||
end
|
||
|
||
desc "Migrate pinch_request_signup to request_signup"
|
||
task(migrate_pinch_request_signup: :environment) do
|
||
count = ChallengeAssignment.where("pinch_request_signup_id IS NOT NULL AND request_signup_id IS NULL").update_all("request_signup_id = pinch_request_signup_id")
|
||
puts("Migrated pinch_request_signup for #{count} challenge assignments.")
|
||
end
|
||
|
||
desc "Reindex tags associated with works that are hidden or unrevealed"
|
||
task(reindex_hidden_unrevealed_tags: :environment) do
|
||
hidden_count = Work.hidden.count
|
||
hidden_batches = (hidden_count + 999) / 1_000
|
||
puts "Inspecting #{hidden_count} hidden works in #{hidden_batches} batches"
|
||
Work.hidden.find_in_batches.with_index do |batch, index|
|
||
batch.each { |work| work.taggings.each(&:update_search) }
|
||
puts "Finished batch #{index + 1} of #{hidden_batches}"
|
||
end
|
||
|
||
unrevealed_count = Work.unrevealed.count
|
||
unrevealed_batches = (unrevealed_count + 999) / 1_000
|
||
puts "Inspecting #{unrevealed_count} unrevealed works in #{unrevealed_batches} batches"
|
||
Work.unrevealed.find_in_batches.with_index do |batch, index|
|
||
batch.each { |work| work.taggings.each(&:update_search) }
|
||
puts "Finished batch #{index + 1} of #{unrevealed_batches}"
|
||
end
|
||
|
||
puts "Finished reindexing tags on hidden and unrevealed works"
|
||
end
|
||
|
||
desc "Convert user kudos from users with the official role to guest kudos"
|
||
task(convert_official_kudos: :environment) do
|
||
official_users = Role.find_by(name: "official")&.users
|
||
if official_users.blank?
|
||
puts "No official users found"
|
||
else
|
||
official_users.each do |user|
|
||
kudos = user.kudos
|
||
next if kudos.blank?
|
||
|
||
puts "Updating #{kudos.size} kudos from #{user.login}"
|
||
user.remove_user_from_kudos
|
||
end
|
||
|
||
puts "Finished converting kudos from official users to guest kudos"
|
||
end
|
||
end
|
||
|
||
desc "Convert user kudos from users with the archivist role to guest kudos"
|
||
task(convert_archivist_kudos: :environment) do
|
||
archivist_users = Role.find_by(name: "archivist")&.users
|
||
if archivist_users.blank?
|
||
puts "No archivist users found"
|
||
else
|
||
archivist_users.each do |user|
|
||
kudos = user.kudos
|
||
next if kudos.blank?
|
||
|
||
puts "Updating #{kudos.size} kudos from #{user.login}"
|
||
user.remove_user_from_kudos
|
||
end
|
||
|
||
puts "Finished converting kudos from archivist users to guest kudos"
|
||
end
|
||
end
|
||
|
||
desc "Create TagSetAssociations for non-canonical tags belonging to canonical fandoms in TagSets"
|
||
task(create_non_canonical_tagset_associations: :environment) do
|
||
# We want to get all set taggings where the tag is not canonical, but has a parent fandom that _is_ canonical.
|
||
# This might be possible with pure Ruby, but unfortunately the parent tag in common_taggings is polymorphic
|
||
# (even though it doesn't need to be), which makes that trickier.
|
||
non_canonicals = SetTagging
|
||
.joins("INNER JOIN `tag_sets` `tag_set` ON `tag_set`.`id` = `set_taggings`.`tag_set_id` INNER JOIN `owned_tag_sets` ON `owned_tag_sets`.`tag_set_id` = `tag_set`.`id` INNER JOIN `tags` `tag` ON `tag`.`id` = `set_taggings`.`tag_id` INNER JOIN `common_taggings` `common_tagging` ON `common_tagging`.`common_tag_id` = `tag`.`id` INNER JOIN `tags` `parent_tag` ON `common_tagging`.`filterable_id` = `parent_tag`.`id`")
|
||
.where("`tag`.`canonical` = FALSE AND `tag`.`type` IN ('Character', 'Relationship') AND `tag_set`.`id` IS NOT NULL AND `parent_tag`.`type` = 'Fandom' AND `parent_tag`.`canonical` = TRUE")
|
||
.distinct
|
||
|
||
non_canonicals.find_in_batches.with_index do |batch, index|
|
||
puts "Creating TagSetAssociations for batch #{index + 1}"
|
||
batch.each do |set_tagging|
|
||
owned_tag_set = set_tagging.tag_set.owned_tag_set
|
||
tag = set_tagging.tag
|
||
|
||
fandoms = tag.fandoms.joins(:set_taggings).where(canonical: true, set_taggings: { tag_set_id: set_tagging.tag_set.id })
|
||
fandoms.find_each do |fandom|
|
||
TagSetAssociation.create!(owned_tag_set: owned_tag_set, tag: tag, parent_tag: fandom)
|
||
rescue ActiveRecord::RecordInvalid
|
||
puts "Association already exists for fandom '#{fandom.name}' and tag '#{tag.name}'"
|
||
end
|
||
end
|
||
end
|
||
end
|
||
# This is the end that you have to put new tasks above.
|
||
end
|