Scrub text before indexing (#2832) (#2833)

refs notch8/hykuup_knapsack#553

Author: laritakr

app/indexers/concerns/hyku_indexing.rb

@@ -3,6 +3,7 @@
 ##
 # A mixin for all additional Hyku applicable indexing; both Valkyrie and ActiveFedora friendly.
 module HykuIndexing
+  include ScrubText
   # TODO: Once we've fully moved to Valkyrie, remove the generate_solr_document and move `#to_solr`
   #      to a more conventional method def (e.g. `def to_solr`).  However, we need to tap into two
   #      different inheritance paths based on ActiveFedora or Valkyrie
@@ -54,7 +55,7 @@ def extract_text_from_plain_text_files(object)
     return [] if members.blank?
 
     text_file_sets = members.select { |fs| fs.file_set? && fs.original_file&.mime_type == 'text/plain' }
-    text_file_sets.map { |fs| fs.original_file&.content }
+    text_file_sets.map { |fs| scrub_text(fs.original_file&.content) }
   end
 
   def extract_text_from_child_works(object)

app/indexers/concerns/scrub_text.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+## remove non-UTF-8 characters from a text string
+module ScrubText
+  def scrub_text(text)
+    text.tr("\n", ' ')
+        .squeeze(' ')
+        .encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+  end
+end

app/indexers/hyku/indexers/file_set_indexer.rb

@@ -4,6 +4,7 @@ module Hyku
   module Indexers
     class FileSetIndexer < Hyrax::Indexers::FileSetIndexer
       include Hyrax::Indexer(:bulkrax_metadata) unless Hyrax.config.flexible?
+      include ScrubText
 
       def to_solr
         return super unless Flipflop.default_pdf_viewer?
@@ -27,9 +28,7 @@ def pdf_text
             pdftotext.read
           end
 
-          text.tr("\n", ' ')
-              .squeeze(' ')
-              .encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') # remove non-UTF-8 characters
+          scrub_text(text)
         rescue Errno::ENOENT => e
           raise e unless e.message.include?("No such file or directory - pdftotext")
           Rails.logger.warn("`pdfinfo' is not installed; unable to extract text from the PDF's content")