diff --git a/lib/data_cleanup.rb b/lib/data_cleanup.rb index 985768b..5f105fd 100644 --- a/lib/data_cleanup.rb +++ b/lib/data_cleanup.rb @@ -1,9 +1,13 @@ # frozen_string_literal: true + +require_relative "data_cleanup/model_check" +require_relative "data_cleanup/instance_check" +require_relative "data_cleanup/reporting" +require_relative "data_cleanup/rules" + module DataCleanup - require_relative "data_cleanup/model_check" - require_relative "data_cleanup/instance_check" - require_relative "data_cleanup/reporting" - require_relative "data_cleanup/rules" + + COLOR_CODES = { red: 31, green: 32 } module_function @@ -11,14 +15,9 @@ @logger ||= Logger.new(Rails.root.join("log", "validations.log")) end - COLOR_CODES = { red: 31, green: 32 } - - def logger.info(message, inline: false, color: nil) - message = message + "\n" unless inline - if color - message = "\e[#{COLOR_CODES[color]}m#{message}\e[0m" - end - super(message) unless inline + def display(message, inline: false, color: nil) + message = "#{message}\n" unless inline + message = "\e[#{COLOR_CODES[color]}m#{message}\e[0m" if color print message end end diff --git a/lib/data_cleanup/instance_check.rb b/lib/data_cleanup/instance_check.rb index c7569b8..02d9c5e 100644 --- a/lib/data_cleanup/instance_check.rb +++ b/lib/data_cleanup/instance_check.rb @@ -6,17 +6,23 @@ # frozen_string_literal: true def call(instance) + DataCleanup.logger.info("Checking #{instance.class}##{instance.id}...") Reporting.total_record_count += 1 begin if instance.invalid? + DataCleanup.logger.info(<<~TEXT) + Instance #{instance.class}##{instance.id} invalid! + Errors: #{instance.errors.full_messages.to_sentence} + TEXT Reporting.invalid_record_count += 1 Reporting.invalid_records << instance - DataCleanup.logger.info("F", inline: true) + DataCleanup.display("F", inline: true) else - DataCleanup.logger.info(".", inline: true) + DataCleanup.logger.info("Instance #{instance.class}##{instance.id} valid!") + DataCleanup.display(".", inline: true) end rescue Dragonfly::Job::Fetch::NotFound - DataCleanup.logger.info(".", inline: true) + DataCleanup.display(".", inline: true) end end end diff --git a/lib/data_cleanup/model_check.rb b/lib/data_cleanup/model_check.rb index 1b42446..b1b793b 100644 --- a/lib/data_cleanup/model_check.rb +++ b/lib/data_cleanup/model_check.rb @@ -19,12 +19,12 @@ when 'EXCLUDE' return if model.model_name.in?(models.split(",")) end - DataCleanup.logger.info "Checking #{model.model_name.plural}:" + DataCleanup.display "Checking #{model.model_name.plural}:" model.find_in_batches do |batch| instance_check = InstanceCheck.new batch.each { |instance| instance_check.(instance) } end - DataCleanup.logger.info "" + DataCleanup.display "" end end end diff --git a/lib/data_cleanup/reporting.rb b/lib/data_cleanup/reporting.rb index ebeba3b..50c13e0 100644 --- a/lib/data_cleanup/reporting.rb +++ b/lib/data_cleanup/reporting.rb @@ -24,9 +24,12 @@ end def report - issues_found.each { |issue| DataCleanup.logger.info issue } + issues_found.each do |issue| + DataCleanup.display issue + DataCleanup.logger.info issue + end color = invalid_record_count.zero? ? :green : :red - DataCleanup.logger.info(<<~TEXT, color: color) + DataCleanup.display(<<~TEXT, color: color) Invalid records: #{invalid_record_count} / #{total_record_count} TEXT end diff --git a/lib/data_cleanup/rules/answer/fix_blank_user.rb b/lib/data_cleanup/rules/answer/fix_blank_user.rb index 987834a..95fc0ba 100644 --- a/lib/data_cleanup/rules/answer/fix_blank_user.rb +++ b/lib/data_cleanup/rules/answer/fix_blank_user.rb @@ -10,8 +10,17 @@ end def call - ::Answer.where(user: nil).destroy_all + ::Answer.where(user: nil) + .includes(plan: { roles: :user }) + .find_in_batches do |answers| + + answers.each do |answer| + log("Updating Answer##{answer.id} with user: #{user.plan.owner}") + answer.update(user: answer.plan.owner) + end + end end + end end end diff --git a/lib/data_cleanup/rules/answer/fix_duplicate_question.rb b/lib/data_cleanup/rules/answer/fix_duplicate_question.rb index 300d4a5..d028842 100644 --- a/lib/data_cleanup/rules/answer/fix_duplicate_question.rb +++ b/lib/data_cleanup/rules/answer/fix_duplicate_question.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + module DataCleanup module Rules # Fix duplicate question on Answer @@ -15,9 +16,10 @@ .count.select { |k,v| v > 1 } # Values looks like [{ [123, 199] => 2}, ...] dataset.each do |values, count| + log("Destroying all Answers that are duplicates of earlier answers on the same question.") # ... and destroy all duplicates, keeping the latest record ::Answer.where(question: values.first, plan_id: values.last) - .order("created_at DESC") + .order("updated_at DESC") .offset(1) .destroy_all end diff --git a/lib/data_cleanup/rules/base.rb b/lib/data_cleanup/rules/base.rb index cff2d1d..cc828e4 100644 --- a/lib/data_cleanup/rules/base.rb +++ b/lib/data_cleanup/rules/base.rb @@ -3,6 +3,10 @@ # Base class for rules to clean invalid database records class Base + def log(message) + DataCleanup.logger.info(message) + end + # Description of the rule and how it's fixing the data def description self.class.name.humanize @@ -12,6 +16,7 @@ def call raise NotImplementedError, "Please define call() in #{self}" end + end end end diff --git a/lib/data_cleanup/rules/guidance_group/fix_blank_optional_subset.rb b/lib/data_cleanup/rules/guidance_group/fix_blank_optional_subset.rb index 9cad432..9ca7498 100644 --- a/lib/data_cleanup/rules/guidance_group/fix_blank_optional_subset.rb +++ b/lib/data_cleanup/rules/guidance_group/fix_blank_optional_subset.rb @@ -8,6 +8,7 @@ end def call + log("Updating all GuidanceGroups") ::GuidanceGroup.where(optional_subset: nil) .update_all(optional_subset: false) end diff --git a/lib/data_cleanup/rules/org/fix_blank_abbreviation.rb b/lib/data_cleanup/rules/org/fix_blank_abbreviation.rb index 40354b6..118eed4 100644 --- a/lib/data_cleanup/rules/org/fix_blank_abbreviation.rb +++ b/lib/data_cleanup/rules/org/fix_blank_abbreviation.rb @@ -13,9 +13,12 @@ def call if File.exists?(YAML_FILE_PATH) YAML.load_file(YAML_FILE_PATH).each do |attributes| - attributes = attributes.with_indifferent_access - ::Org.where(name: attributes["name"], id: attributes['id']) - .update_all(abbreviation: attributes['abbreviation']) + attributes = attributes.with_indifferent_access + id = attributes['id'] + name = attributes['name'] + abbreviation = attributes['abbreviation'] + log("Adding abbreviation #{abbreviation} to Org '#{name}'") + ::Org.where(name: name, id: id).update_all(abbreviation: abbreviation) end else raise "Please create a YAML file at #{YAML_FILE_PATH}" diff --git a/lib/data_cleanup/rules/org/fix_blank_feedback_email_msg.rb b/lib/data_cleanup/rules/org/fix_blank_feedback_email_msg.rb index cefdc38..2442525 100644 --- a/lib/data_cleanup/rules/org/fix_blank_feedback_email_msg.rb +++ b/lib/data_cleanup/rules/org/fix_blank_feedback_email_msg.rb @@ -17,6 +17,8 @@ end def call + ids = ::Org.where(feedback_enabled: true, feedback_email_msg: "").pluck(:id) + log("Adding default feedback_enabled for orgs: #{ids}") ::Org.where(feedback_enabled: true, feedback_email_msg: "") .update_all(feedback_email_msg: DEFAULT_MSG) end diff --git a/lib/data_cleanup/rules/org/fix_blank_feedback_email_subject.rb b/lib/data_cleanup/rules/org/fix_blank_feedback_email_subject.rb index e190b22..af87b5d 100644 --- a/lib/data_cleanup/rules/org/fix_blank_feedback_email_subject.rb +++ b/lib/data_cleanup/rules/org/fix_blank_feedback_email_subject.rb @@ -10,8 +10,10 @@ end def call + ids = ::Org.where(feedback_enabled: true, feedback_email_subject: "").pluck(:id) + log("Adding default feedback_email_subject for orgs: #{ids}") ::Org.where(feedback_enabled: true, feedback_email_subject: "") - .update_all(feedback_email_subject: DEFAULT_SUBJECT) + .update_all(feedback_email_subject: DEFAULT_SUBJECT) end end end diff --git a/lib/data_cleanup/rules/org/fix_blank_language.rb b/lib/data_cleanup/rules/org/fix_blank_language.rb index 14ba0ff..eb43178 100644 --- a/lib/data_cleanup/rules/org/fix_blank_language.rb +++ b/lib/data_cleanup/rules/org/fix_blank_language.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DataCleanup module Rules module Org @@ -10,6 +12,8 @@ end def call + ids = Org.where(language: nil).pluck(:id) + log("Setting language to #{DEFAULT_LANGUAGE} for Orgs: #{ids}") ::Org.where(language: nil).update_all(language_id: DEFAULT_LANGUAGE.id) end end diff --git a/lib/data_cleanup/rules/phase/fix_duplicate_number.rb b/lib/data_cleanup/rules/phase/fix_duplicate_number.rb index e733dcd..2467211 100644 --- a/lib/data_cleanup/rules/phase/fix_duplicate_number.rb +++ b/lib/data_cleanup/rules/phase/fix_duplicate_number.rb @@ -19,6 +19,7 @@ .order("number ASC, created_at ASC") .pluck(:id) template = ::Template.find(template_id) + log("Reordering Phase number within Template##{template.id}") ::Phase.update_numbers!(*ids, parent: template) end end diff --git a/lib/data_cleanup/rules/plan/fix_blank_title.rb b/lib/data_cleanup/rules/plan/fix_blank_title.rb index 8dc87a9..d0a08df 100644 --- a/lib/data_cleanup/rules/plan/fix_blank_title.rb +++ b/lib/data_cleanup/rules/plan/fix_blank_title.rb @@ -8,7 +8,9 @@ end def call - ::Plan.where(title: [nil, '']).each do |plan| + ids = ::Plan.where(title: [nil, '']).ids + ::Plan.find(ids).each do |plan| + info("Adding default title to Plan##{plan.id}") plan.update(title: "My plan (#{plan.template.title})") end end diff --git a/lib/data_cleanup/rules/question/fix_duplicate_number.rb b/lib/data_cleanup/rules/question/fix_duplicate_number.rb index 083b8fb..65fb88d 100644 --- a/lib/data_cleanup/rules/question/fix_duplicate_number.rb +++ b/lib/data_cleanup/rules/question/fix_duplicate_number.rb @@ -19,6 +19,7 @@ .order("number ASC, created_at ASC") .pluck(:id) section = ::Section.find(section_id) + log("Reordering Question number in Section##{section.id}") ::Question.update_numbers!(*ids, parent: section) end end diff --git a/lib/data_cleanup/rules/question_format/fix_blank_description.rb b/lib/data_cleanup/rules/question_format/fix_blank_description.rb index 7c8f06c..cda21e1 100644 --- a/lib/data_cleanup/rules/question_format/fix_blank_description.rb +++ b/lib/data_cleanup/rules/question_format/fix_blank_description.rb @@ -11,6 +11,7 @@ def call ::QuestionFormat.where(description: "").each do |qf| + log("Adding default description to QuestionFormat##{qf.id}") qf.update!(description: "#{qf.title} format") end end diff --git a/lib/data_cleanup/rules/region/fix_blank_description.rb b/lib/data_cleanup/rules/region/fix_blank_description.rb index 1ddee40..b5389fa 100644 --- a/lib/data_cleanup/rules/region/fix_blank_description.rb +++ b/lib/data_cleanup/rules/region/fix_blank_description.rb @@ -9,6 +9,7 @@ def call ::Region.where(description: [nil, '']).each do |region| + log("Adding default description to Region##{region.id}") region.update!(description: "#{region.name} region") end end diff --git a/lib/data_cleanup/rules/role/fix_blank_plan.rb b/lib/data_cleanup/rules/role/fix_blank_plan.rb index 3e4219b..757b084 100644 --- a/lib/data_cleanup/rules/role/fix_blank_plan.rb +++ b/lib/data_cleanup/rules/role/fix_blank_plan.rb @@ -10,7 +10,9 @@ end def call - ::Role.where(plan: nil).destroy_all + ids = ::Role.where(plan: nil).ids + log("Destroying Roles without Plan: #{ids}") + ::Role.destroy(ids) end end end diff --git a/lib/data_cleanup/rules/section/fix_duplicate_number.rb b/lib/data_cleanup/rules/section/fix_duplicate_number.rb index c2c41c8..f71369a 100644 --- a/lib/data_cleanup/rules/section/fix_duplicate_number.rb +++ b/lib/data_cleanup/rules/section/fix_duplicate_number.rb @@ -19,6 +19,7 @@ .order("number ASC, created_at ASC") .pluck(:id) phase = ::Phase.find(phase_id) + log("Re-setting number order for Sections in Phase##{phase.id}") ::Section.update_numbers!(*ids, parent: phase) end end diff --git a/lib/data_cleanup/rules/template/fix_blank_locale.rb b/lib/data_cleanup/rules/template/fix_blank_locale.rb index 4fdd5e7..786a7be 100644 --- a/lib/data_cleanup/rules/template/fix_blank_locale.rb +++ b/lib/data_cleanup/rules/template/fix_blank_locale.rb @@ -8,8 +8,9 @@ end def call - ::Template.where(locale: nil) - .update_all(locale: FastGettext.default_locale) + ids = ::Template.where(locale: nil).ids + log("Setting locale to #{FastGettext.default_locale} for Templates #{ids}") + ::Template.where(id: ids).update_all(locale: FastGettext.default_locale) end end end diff --git a/lib/data_cleanup/rules/user_identifier/fix_blank_user.rb b/lib/data_cleanup/rules/user_identifier/fix_blank_user.rb index 9676652..f012258 100644 --- a/lib/data_cleanup/rules/user_identifier/fix_blank_user.rb +++ b/lib/data_cleanup/rules/user_identifier/fix_blank_user.rb @@ -10,7 +10,9 @@ end def call - ::UserIdentifier.where(user: nil).destroy_all + ids = ::UserIdentifier.where(user: nil).ids + log("Destroying UserIdentifier where ids: #{ids} (no user)") + ::UserIdentifier.destroy(ids) end end end diff --git a/lib/tasks/heal_invalid_records.rake b/lib/tasks/heal_invalid_records.rake index 18a017f..9842327 100644 --- a/lib/tasks/heal_invalid_records.rake +++ b/lib/tasks/heal_invalid_records.rake @@ -4,6 +4,7 @@ desc "Check each record on the DB is valid and report" task :find_invalid_records => :environment do + DataCleanup.logger.info("\n== Finding invalid records =======================\n") models.each do |model| DataCleanup::ModelCheck.new(model).call end @@ -13,6 +14,7 @@ desc "Clean invalid records on the database" task :clean_invalid_records => :environment do + DataCleanup.logger.info("\n== Cleaning invalid records =======================\n") Dir[rule_paths].each do |rule_path| load rule_path klass_name = rule_path.split("rules/").last.gsub(".rb", '').classify