namespace :After do # Keep only the most recent tasks, i.e., about two years' worth. # If you need older tasks, check GitHub. desc "Update the mapping for the work index" task(update_work_mapping: :environment) do WorkIndexer.create_mapping end desc "Fix tags with extra spaces" task(fix_tags_with_extra_spaces: :environment) do total_tags = Tag.count total_batches = (total_tags + 999) / 1000 puts "Inspecting #{total_tags} tags in #{total_batches} batches" report_string = ["Tag ID", "Old tag name", "New tag name"].to_csv Tag.find_in_batches.with_index do |batch, index| batch_number = index + 1 progress_msg = "Batch #{batch_number} of #{total_batches} complete" batch.each do |tag| next unless tag.name != tag.name.squish old_tag_name = tag.name new_tag_name = old_tag_name.gsub(/[[:space:]]/, "_") new_tag_name << "_" while Tag.find_by(name: new_tag_name) tag.update_attribute(:name, new_tag_name) report_row = [tag.id, old_tag_name, new_tag_name].to_csv report_string += report_row end puts(progress_msg) && STDOUT.flush end puts(report_string) && STDOUT.flush end desc "Fix works imported with a noncanonical Teen & Up Audiences rating tag" task(fix_teen_and_up_imported_rating: :environment) do borked_rating_tag = Rating.find_by!(name: "Teen & Up Audiences") canonical_rating_tag = Rating.find_by!(name: ArchiveConfig.RATING_TEEN_TAG_NAME) work_ids = [] invalid_work_ids = [] borked_rating_tag.works.find_each do |work| work.ratings << canonical_rating_tag work.ratings = work.ratings - [borked_rating_tag] if work.save work_ids << work.id else invalid_work_ids << work.id end print(".") && STDOUT.flush end unless work_ids.empty? puts "Converted '#{borked_rating_tag.name}' rating tag on #{work_ids.size} works:" puts work_ids.join(", ") STDOUT.flush end unless invalid_work_ids.empty? puts "The following #{invalid_work_ids.size} works failed validations and could not be saved:" puts invalid_work_ids.join(", ") STDOUT.flush end end desc "Clean up multiple rating tags" task(clean_up_multiple_ratings: :environment) do default_rating_tag = Rating.find_by!(name: ArchiveConfig.RATING_DEFAULT_TAG_NAME) es_results = $elasticsearch.search(index: WorkIndexer.index_name, body: { query: { bool: { filter: { script: { script: { source: "doc['rating_ids'].length > 1", lang: "painless" } } } } } }) invalid_works = QueryResult.new("Work", es_results) puts "There are #{invalid_works.size} works with multiple ratings." fixed_work_ids = [] unfixed_word_ids = [] invalid_works.each do |work| work.ratings = [default_rating_tag] work.rating_string = default_rating_tag.name if work.save fixed_work_ids << work.id else unfixed_word_ids << work.id end print(".") && $stdout.flush end unless fixed_work_ids.empty? puts "Cleaned up having multiple ratings on #{fixed_work_ids.size} works:" puts fixed_work_ids.join(", ") $stdout.flush end unless unfixed_word_ids.empty? puts "The following #{unfixed_word_ids.size} works failed validations and could not be saved:" puts unfixed_word_ids.join(", ") $stdout.flush end end desc "Clean up noncanonical rating tags" task(clean_up_noncanonical_ratings: :environment) do canonical_not_rated_tag = Rating.find_by!(name: ArchiveConfig.RATING_DEFAULT_TAG_NAME) noncanonical_ratings = Rating.where(canonical: false) puts "There are #{noncanonical_ratings.size} noncanonical rating tags." next if noncanonical_ratings.empty? puts "The following noncanonical Ratings will be changed into Additional Tags:" puts noncanonical_ratings.map(&:name).join("\n") work_ids = [] invalid_work_ids = [] noncanonical_ratings.find_each do |tag| works_using_tag = tag.works tag.update_attribute(:type, "Freeform") works_using_tag.find_each do |work| next unless work.ratings.empty? work.ratings = [canonical_not_rated_tag] if work.save work_ids << work.id else invalid_work_ids << work.id end print(".") && STDOUT.flush end end unless work_ids.empty? puts "The following #{work_ids.size} works were left without a rating and successfully received the Not Rated rating:" puts work_ids.join(", ") STDOUT.flush end unless invalid_work_ids.empty? puts "The following #{invalid_work_ids.size} works failed validations and could not be saved:" puts invalid_work_ids.join(", ") STDOUT.flush end end desc "Clean up noncanonical category tags" task(clean_up_noncanonical_categories: :environment) do Category.where(canonical: false).find_each do |tag| tag.update_attribute(:type, "Freeform") puts "Noncanonical Category tag '#{tag.name}' was changed into an Additional Tag." end STDOUT.flush end desc "Add default rating to works missing a rating" task(add_default_rating_to_works: :environment) do work_count = Work.count total_batches = (work_count + 999) / 1000 puts("Checking #{work_count} works in #{total_batches} batches") && STDOUT.flush updated_works = [] Work.find_in_batches.with_index do |batch, index| batch_number = index + 1 batch.each do |work| next unless work.ratings.empty? work.ratings << Rating.find_by!(name: ArchiveConfig.RATING_DEFAULT_TAG_NAME) work.save updated_works << work.id end puts("Batch #{batch_number} of #{total_batches} complete") && STDOUT.flush end puts("Added default rating to works: #{updated_works}") && STDOUT.flush end desc "Backfill renamed_at for existing users" task(add_renamed_at_from_log: :environment) do total_users = User.all.size total_batches = (total_users + 999) / 1000 puts "Updating #{total_users} users in #{total_batches} batches" User.find_in_batches.with_index do |batch, index| batch.each do |user| renamed_at_from_log = user.log_items.where(action: ArchiveConfig.ACTION_RENAME).last&.created_at next unless renamed_at_from_log user.update_column(:renamed_at, renamed_at_from_log) end batch_number = index + 1 progress_msg = "Batch #{batch_number} of #{total_batches} complete" puts(progress_msg) && STDOUT.flush end puts && STDOUT.flush end desc "Fix threads for comments from 2009" task(fix_2009_comment_threads: :environment) do def fix_comment(comment) comment.with_lock do if comment.reply_comment? comment.update_column(:thread, comment.commentable.thread) else comment.update_column(:thread, comment.id) end comment.comments.each { |reply| fix_comment(reply) } end end incorrect = Comment.top_level.where("thread != id") total = incorrect.count puts "Updating #{total} thread(s)" incorrect.find_each.with_index do |comment, index| fix_comment(comment) puts "Fixed thread #{index + 1} out of #{total}" if index % 100 == 99 end end desc "Remove translation_admin role" task(remove_translation_admin_role: :environment) do r = Role.find_by(name: "translation_admin") r&.destroy end desc "Remove full-width and ideographic commas from tags" task(remove_invalid_commas_from_tags: :environment) do puts("Tags can only be renamed by an admin, who will be listed as the tag's last wrangler. Enter the admin login we should use:") login = $stdin.gets.chomp.strip admin = Admin.find_by(login: login) if admin.present? User.current_user = admin [",", "、"].each do |comma| tags = Tag.where("name LIKE ?", "%#{comma}%") tags.each do |tag| new_name = tag.name.gsub(/#{comma}/, "") if tag.update(name: new_name) || tag.update(name: "#{new_name} - AO3-6626") puts(tag.reload.name) else puts("Could not rename #{tag.reload.name}") end $stdout.flush end end else puts("Admin not found.") end end desc "Add suffix to existing Underage Sex tag in preparation for Underage warning rename" task(add_suffix_to_underage_sex_tag: :environment) do puts("Tags can only be renamed by an admin, who will be listed as the tag's last wrangler. Enter the admin login we should use:") login = $stdin.gets.chomp.strip admin = Admin.find_by(login: login) if admin.present? User.current_user = admin tag = Tag.find_by_name("Underage Sex") if tag.blank? puts("No Underage Sex tag found.") elsif tag.is_a?(ArchiveWarning) puts("Underage Sex is already an Archive Warning.") else suffixed_name = "Underage Sex - #{tag.class}" if tag.update(name: suffixed_name) puts("Renamed Underage Sex tag to #{tag.reload.name}.") else puts("Failed to rename Underage Sex tag to #{suffixed_name}.") end $stdout.flush end else puts("Admin not found.") end end desc "Rename Underage warning to Underage Sex" task(rename_underage_warning: :environment) do puts("Tags can only be renamed by an admin, who will be listed as the tag's last wrangler. Enter the admin login we should use:") login = $stdin.gets.chomp.strip admin = Admin.find_by(login: login) if admin.present? User.current_user = admin tag = ArchiveWarning.find_by_name("Underage") if tag.blank? puts("No Underage warning tag found.") else new_name = "Underage Sex" if tag.update(name: new_name) puts("Renamed Underage warning tag to #{tag.reload.name}.") else puts("Failed to rename Underage warning tag to #{new_name}.") end $stdout.flush end else puts("Admin not found.") end end desc "Migrate collection icons to ActiveStorage paths" task(migrate_collection_icons: :environment) do require "aws-sdk-s3" require "open-uri" return unless Rails.env.staging? || Rails.env.production? bucket_name = ENV["S3_BUCKET"] prefix = "collections/icons/" s3 = Aws::S3::Resource.new( region: ENV["S3_REGION"], access_key_id: ENV["S3_ACCESS_KEY_ID"], secret_access_key: ENV["S3_SECRET_ACCESS_KEY"] ) old_bucket = s3.bucket(bucket_name) new_bucket = s3.bucket(ENV["TARGET_BUCKET"]) Collection.no_touching do old_bucket.objects(prefix: prefix).each do |object| # Path example: staging/icons/108621/original.png path_parts = object.key.split("/") next unless path_parts[-1]&.include?("original") next if ActiveStorage::Attachment.where(record_type: "Collection", record_id: path_parts[-2]).any? collection_id = path_parts[-2] old_icon = URI.open("https://s3.amazonaws.com/#{bucket_name}/#{object.key}") checksum = OpenSSL::Digest.new("MD5").tap do |result| while (chunk = old_icon.read(5.megabytes)) result << chunk end old_icon.rewind end.base64digest key = nil ActiveRecord::Base.transaction do blob = ActiveStorage::Blob.create_before_direct_upload!( filename: path_parts[-1], byte_size: old_icon.size, checksum: checksum, content_type: Marcel::MimeType.for(old_icon) ) key = blob.key blob.attachments.create( name: "icon", record_type: "Collection", record_id: collection_id ) end new_bucket.put_object(key: key, body: old_icon, acl: "bucket-owner-full-control") puts "Finished collection #{collection_id}" $stdout.flush end end end desc "Migrate pseud icons to ActiveStorage paths" task(migrate_pseud_icons: :environment) do require "aws-sdk-s3" require "open-uri" return unless Rails.env.staging? || Rails.env.production? bucket_name = ENV["S3_BUCKET"] prefix = Rails.env.production? ? "icons/" : "staging/icons/" s3 = Aws::S3::Resource.new( region: ENV["S3_REGION"], access_key_id: ENV["S3_ACCESS_KEY_ID"], secret_access_key: ENV["S3_SECRET_ACCESS_KEY"] ) old_bucket = s3.bucket(bucket_name) new_bucket = s3.bucket(ENV["TARGET_BUCKET"]) Pseud.no_touching do old_bucket.objects(prefix: prefix).each do |object| # Path example: staging/icons/108621/original.png path_parts = object.key.split("/") next unless path_parts[-1]&.include?("original") next if ActiveStorage::Attachment.where(record_type: "Pseud", record_id: path_parts[-2]).any? pseud_id = path_parts[-2] old_icon = URI.open("https://s3.amazonaws.com/#{bucket_name}/#{object.key}") checksum = OpenSSL::Digest.new("MD5").tap do |result| while (chunk = old_icon.read(5.megabytes)) result << chunk end old_icon.rewind end.base64digest key = nil ActiveRecord::Base.transaction do blob = ActiveStorage::Blob.create_before_direct_upload!( filename: path_parts[-1], byte_size: old_icon.size, checksum: checksum, content_type: Marcel::MimeType.for(old_icon) ) key = blob.key blob.attachments.create( name: "icon", record_type: "Pseud", record_id: pseud_id ) end new_bucket.put_object(key: key, body: old_icon, acl: "bucket-owner-full-control") puts "Finished pseud #{pseud_id}" $stdout.flush end end end desc "Migrate skin icons to ActiveStorage paths" task(migrate_skin_icons: :environment) do require "aws-sdk-s3" require "open-uri" return unless Rails.env.staging? || Rails.env.production? bucket_name = ENV["S3_BUCKET"] prefix = "skins/icons/" s3 = Aws::S3::Resource.new( region: ENV["S3_REGION"], access_key_id: ENV["S3_ACCESS_KEY_ID"], secret_access_key: ENV["S3_SECRET_ACCESS_KEY"] ) old_bucket = s3.bucket(bucket_name) new_bucket = s3.bucket(ENV["TARGET_BUCKET"]) Skin.no_touching do old_bucket.objects(prefix: prefix).each do |object| # Path example: staging/icons/108621/original.png path_parts = object.key.split("/") next unless path_parts[-1]&.include?("original") next if ActiveStorage::Attachment.where(record_type: "Skin", record_id: path_parts[-2]).any? skin_id = path_parts[-2] old_icon = URI.open("https://s3.amazonaws.com/#{bucket_name}/#{object.key}") checksum = OpenSSL::Digest.new("MD5").tap do |result| while (chunk = old_icon.read(5.megabytes)) result << chunk end old_icon.rewind end.base64digest key = nil ActiveRecord::Base.transaction do blob = ActiveStorage::Blob.create_before_direct_upload!( filename: path_parts[-1], byte_size: old_icon.size, checksum: checksum, content_type: Marcel::MimeType.for(old_icon) ) key = blob.key blob.attachments.create( name: "icon", record_type: "Skin", record_id: skin_id ) end new_bucket.put_object(key: key, body: old_icon, acl: "bucket-owner-full-control") puts "Finished skin #{skin_id}" $stdout.flush end end end desc "Migrate pinch_request_signup to request_signup" task(migrate_pinch_request_signup: :environment) do count = ChallengeAssignment.where("pinch_request_signup_id IS NOT NULL AND request_signup_id IS NULL").update_all("request_signup_id = pinch_request_signup_id") puts("Migrated pinch_request_signup for #{count} challenge assignments.") end desc "Reindex tags associated with works that are hidden or unrevealed" task(reindex_hidden_unrevealed_tags: :environment) do hidden_count = Work.hidden.count hidden_batches = (hidden_count + 999) / 1_000 puts "Inspecting #{hidden_count} hidden works in #{hidden_batches} batches" Work.hidden.find_in_batches.with_index do |batch, index| batch.each { |work| work.taggings.each(&:update_search) } puts "Finished batch #{index + 1} of #{hidden_batches}" end unrevealed_count = Work.unrevealed.count unrevealed_batches = (unrevealed_count + 999) / 1_000 puts "Inspecting #{unrevealed_count} unrevealed works in #{unrevealed_batches} batches" Work.unrevealed.find_in_batches.with_index do |batch, index| batch.each { |work| work.taggings.each(&:update_search) } puts "Finished batch #{index + 1} of #{unrevealed_batches}" end puts "Finished reindexing tags on hidden and unrevealed works" end desc "Convert user kudos from users with the official role to guest kudos" task(convert_official_kudos: :environment) do official_users = Role.find_by(name: "official")&.users if official_users.blank? puts "No official users found" else official_users.each do |user| kudos = user.kudos next if kudos.blank? puts "Updating #{kudos.size} kudos from #{user.login}" user.remove_user_from_kudos end puts "Finished converting kudos from official users to guest kudos" end end desc "Convert user kudos from users with the archivist role to guest kudos" task(convert_archivist_kudos: :environment) do archivist_users = Role.find_by(name: "archivist")&.users if archivist_users.blank? puts "No archivist users found" else archivist_users.each do |user| kudos = user.kudos next if kudos.blank? puts "Updating #{kudos.size} kudos from #{user.login}" user.remove_user_from_kudos end puts "Finished converting kudos from archivist users to guest kudos" end end desc "Create TagSetAssociations for non-canonical tags belonging to canonical fandoms in TagSets" task(create_non_canonical_tagset_associations: :environment) do # We want to get all set taggings where the tag is not canonical, but has a parent fandom that _is_ canonical. # This might be possible with pure Ruby, but unfortunately the parent tag in common_taggings is polymorphic # (even though it doesn't need to be), which makes that trickier. non_canonicals = SetTagging .joins("INNER JOIN `tag_sets` `tag_set` ON `tag_set`.`id` = `set_taggings`.`tag_set_id` INNER JOIN `owned_tag_sets` ON `owned_tag_sets`.`tag_set_id` = `tag_set`.`id` INNER JOIN `tags` `tag` ON `tag`.`id` = `set_taggings`.`tag_id` INNER JOIN `common_taggings` `common_tagging` ON `common_tagging`.`common_tag_id` = `tag`.`id` INNER JOIN `tags` `parent_tag` ON `common_tagging`.`filterable_id` = `parent_tag`.`id`") .where("`tag`.`canonical` = FALSE AND `tag`.`type` IN ('Character', 'Relationship') AND `tag_set`.`id` IS NOT NULL AND `parent_tag`.`type` = 'Fandom' AND `parent_tag`.`canonical` = TRUE") .distinct non_canonicals.find_in_batches.with_index do |batch, index| puts "Creating TagSetAssociations for batch #{index + 1}" batch.each do |set_tagging| owned_tag_set = set_tagging.tag_set.owned_tag_set tag = set_tagging.tag fandoms = tag.fandoms.joins(:set_taggings).where(canonical: true, set_taggings: { tag_set_id: set_tagging.tag_set.id }) fandoms.find_each do |fandom| TagSetAssociation.create!(owned_tag_set: owned_tag_set, tag: tag, parent_tag: fandom) rescue ActiveRecord::RecordInvalid puts "Association already exists for fandom '#{fandom.name}' and tag '#{tag.name}'" end end end end # This is the end that you have to put new tasks above. end