Merge pull request #686 from IU-Libraries-Joint-Development/essi-2130_remote_file_retrieval

[ESSI-2130] apply Fedora/S3 file retrieval for OCR generation, characterization, PDF generation, download

Author: aploshay

app/jobs/characterize_job.rb

@@ -10,16 +10,9 @@ class CharacterizeJob < ApplicationJob
   def perform(file_set, file_id, filepath = nil, derivation_path = nil, delete_characterization_path = false)
     raise "#{file_set.class.characterization_proxy} was not found for FileSet #{file_set.id}" unless file_set.characterization_proxy?
     unless filepath && File.exist?(filepath)
-      if file_set.content_location&.start_with?('s3://') # Ensure external file is available locally
-        ext_id = file_set.content_location.split('/').last
-        ext_resp = ESSI.external_storage.get(ext_id)
-        filepath = Hyrax::WorkingDirectory.send(:copy_stream_to_working_directory, ext_id, ext_id, ext_resp.body)
-        delete_characterization_path = true
-      else
-        filepath = Hyrax::WorkingDirectory.find_or_retrieve(file_id, file_set.id) unless filepath && File.exist?(filepath)
-        delete_characterization_path = false
-      end
+      delete_characterization_path = file_set.external?
     end
+    filepath = file_set.find_or_retrieve(file_id: file_id, filepath: filepath)
     Hydra::Works::CharacterizationService.run(file_set.characterization_proxy, filepath)
     Rails.logger.debug "Ran characterization on #{file_set.characterization_proxy.id} (#{file_set.characterization_proxy.mime_type})"
     file_set.characterization_proxy.save!

app/jobs/create_ocr_job.rb

@@ -6,11 +6,12 @@ class CreateOCRJob < CreateDerivativesJob
   # @param [String, NilClass] filepath the cached file within the Hyrax.config.working_path
   def perform(file_set, file_id, filepath = nil)
     return if file_set.video? && !Hyrax.config.enable_ffmpeg
-    filename = Hyrax::WorkingDirectory.find_or_retrieve(file_id, file_set.id, filepath)
+
+    filepath = file_set.find_or_retrieve(file_id: file_id, filepath: filepath)
 
     # using #create_derivatives instead of #create_ocr_derivatives to use IIIF Print
     # @see https://github.com/scientist-softserv/iiif_print/blob/d14246664048c708071c7ff4de2e9a34aa703465/app/services/iiif_print/pluggable_derivative_service.rb#L25
-    file_set.send(:file_set_derivatives_service).send(:create_derivatives, filename)
+    file_set.send(:file_set_derivatives_service).send(:create_derivatives, filepath)
 
     # Reload from Fedora and reindex for thumbnail and extracted text
     file_set.reload

app/models/file_set.rb

@@ -29,4 +29,23 @@ def ocr_language
   def original_file_id
     @original_file_id ||= self.original_file&.id
   end
+
+  def external?
+    ESSI.external_storage.external?(self)
+  end
+
+  def external_id
+    @external_id ||= ESSI.external_storage.external_id(self)
+  end
+
+  # supplement to Hyrax::WorkingDirectory.find_or_retrieve, but aware of external storage
+  def find_or_retrieve(file_id: original_file&.id, filepath: nil)
+    return filepath if filepath && File.exist?(filepath)
+    if self.external?
+      filepath = ESSI.external_storage.find_or_retrieve(self, file_id: file_id, filepath: filepath)
+    else
+      filepath = Hyrax::WorkingDirectory.find_or_retrieve(file_id, self.id, filepath: filepath)
+    end
+    return filepath
+  end
 end

app/services/essi/external_storage_service.rb

@@ -95,6 +95,28 @@ def prefix_id(id)
       "#{@prefix}/#{treeify_id(id)}"
     end
 
+    def external?(file_set)
+      file_set.content_location&.start_with?('s3://') || false
+    end
+
+    def external_id(file_set)
+      file_set.content_location.split('/').last if external?(file_set)
+    end
+
+    # external equivalent to Hyrax::WorkingDirectory.find_or_retrieve
+    def find_or_retrieve(file_set, file_id: file_set.original_file&.id, filepath: nil)
+      return filepath if filepath && File.exist?(filepath)
+      if external?(file_set)
+        ext_id = external_id(file_set)
+        ext_resp = get(ext_id)
+        filepath = Hyrax::WorkingDirectory.send(:copy_stream_to_working_directory, ext_id, ext_id, ext_resp.body)
+      else
+        Rails.logger.warn("External storage find_or_retrieve called for FileSet #{file_set.id} not stored externally")
+        filepath = nil
+      end
+      return filepath
+    end
+
   private
 
     def endpoint

app/services/essi/generate_pdf_service.rb

@@ -73,7 +73,7 @@ def render_dimensions(image_width)
     def generate_width(file_set_id)
       begin
         file_set = FileSet.find(file_set_id)
-        filepath = Hyrax::WorkingDirectory.find_or_retrieve(file_set.original_file.id, file_set.id)
+        filepath = file_set.find_or_retrieve
         terms = Hydra::Works::CharacterizationService.run(file_set.original_file, filepath)
         CharacterizeJob.perform_later(file_set, file_set.original_file.id)
       rescue

lib/extensions/hyrax/downloads_controller/external_storage.rb

@@ -6,8 +6,8 @@ module ExternalStorage
         def show
           if params[:file] && params[:file] == 'extracted_text'
             super
-          elsif asset.content_location&.start_with?('s3://')
-            ext_id = asset.content_location.split('/').last
+          elsif asset.try(:external?)
+            ext_id = asset.external_id
             external_asset = ESSI.external_storage.get(ext_id)
             send_data external_asset.body.read, filename: ext_id
           else

spec/models/file_set_spec.rb

@@ -36,4 +36,78 @@
       end
     end
   end
+
+  let(:external_id) { "s3_id" }
+  let(:external_location) { "s3://#{external_id}" }
+
+  describe "#external?" do
+    context "when stored in S3" do
+      before { allow(file_set).to receive(:content_location).and_return(external_location) }
+      it "returns true" do
+        expect(file_set.external?).to eq true
+      end
+    end
+    context "when stored in Fedora" do
+      it "returns false" do
+        expect(file_set.external?).to eq false
+      end
+    end
+  end
+
+  describe "#external_id" do
+    context "when stored in S3" do
+      before { allow(file_set).to receive(:content_location).and_return(external_location) }
+      it "returns the S3 internal id" do
+        expect(file_set.external_id).to eq external_id
+      end
+    end
+    context "when stored in Fedora" do
+      it "returns nil" do
+        expect(file_set.external_id).to be_nil
+      end
+    end
+  end
+
+  describe "#find_or_retrieve" do
+    shared_examples "find_or_retrieve examples" do |argument_filepath|
+      context "when file is stored in S3" do
+        before { allow(file_set).to receive(:content_location).and_return(external_location) }
+        let(:output_filepath) { 'filepath_from_s3' }
+        before { allow(ESSI.external_storage).to receive(:find_or_retrieve).and_return(output_filepath) }
+        it "calls ESSI.external_storage.find_or_retrieve" do
+          expect(ESSI.external_storage).to receive(:find_or_retrieve)
+          file_set.find_or_retrieve(filepath: argument_filepath)
+        end
+        it "returns filepath" do
+          expect(file_set.find_or_retrieve(filepath: argument_filepath)).to eq output_filepath
+        end
+      end
+      context "when file is stored in Fedora" do
+        let(:output_filepath) { 'filepath_from_fedora' }
+        before { allow(Hyrax::WorkingDirectory).to receive(:find_or_retrieve).and_return(output_filepath) }
+        it "calls Hyrax::WorkingDirectory.find_or_retrieve" do
+          expect(Hyrax::WorkingDirectory).to receive(:find_or_retrieve)
+          file_set.find_or_retrieve(filepath: argument_filepath)
+        end
+        it "returns filepath" do
+          expect(file_set.find_or_retrieve(filepath: argument_filepath)).to eq output_filepath
+        end
+      end
+    end
+    context "when filepath provided" do
+      let(:argument_filepath) { '/tmp/existing_file.txt' }
+      context "when file present" do
+        before { allow(File).to receive(:exist?).with(argument_filepath).and_return(true) }
+        it "returns the filepath" do
+          expect(file_set.find_or_retrieve(filepath: argument_filepath)).to eq argument_filepath
+        end
+      end
+      context "when file absent" do
+        include_examples "find_or_retrieve examples", '/tmp/existing_file.txt'
+      end
+    end
+    context "when filepath not provided" do
+      include_examples "find_or_retrieve examples", nil
+    end
+  end
 end

spec/services/essi/external_storage_service_spec.rb

@@ -42,4 +42,83 @@
       expect(service.prefix_id(id)).to eq('ext-store/es/si/-e/xt/essi-ext-store-spec')
     end
   end
+
+  let(:file_set) { FactoryBot.build(:file_set) }
+  let(:external_id) { "s3_id" }
+  let(:external_location) { "s3://#{external_id}" }
+  describe "#external?" do
+    context "when stored in S3" do
+      before { allow(file_set).to receive(:content_location).and_return(external_location) }
+      it "returns true" do
+        expect(service.external?(file_set)).to eq true
+      end
+    end
+    context "when stored in Fedora" do
+      it "returns false" do
+        expect(service.external?(file_set)).to eq false
+      end
+    end
+  end
+
+  describe "#external_id" do
+    context "when stored in S3" do
+      before { allow(file_set).to receive(:content_location).and_return(external_location) }
+      it "returns the S3 internal id" do
+        expect(service.external_id(file_set)).to eq external_id
+      end
+    end
+    context "when stored in Fedora" do
+      it "returns nil" do
+        expect(file_set.external_id).to be_nil
+      end
+    end
+  end
+
+  describe "#find_or_retrieve" do
+    shared_examples "find_or_retrieve examples" do |argument_filepath|
+      context "when file is stored in S3" do
+        let(:output_filepath) { 'filepath_from_s3' }
+        before do
+          allow(file_set).to receive(:content_location).and_return("s3://server/external_id")
+          allow(service).to receive(:get).and_return(double(body: nil))
+          allow(Hyrax::WorkingDirectory).to receive(:copy_stream_to_working_directory).and_return(output_filepath)
+        end
+        it "retrieves external file content" do
+          expect(service).to receive(:get)
+          service.find_or_retrieve(file_set, filepath: argument_filepath)
+        end
+        it "copies stream to working directory" do
+          expect(Hyrax::WorkingDirectory).to receive(:copy_stream_to_working_directory)
+          service.find_or_retrieve(file_set, filepath: argument_filepath)
+        end
+        it "returns filepath" do
+          expect(service.find_or_retrieve(file_set, filepath: argument_filepath)).to eq output_filepath
+        end
+      end
+      context "when file is stored in Fedora" do
+        it "logs warning" do
+          expect(Rails.logger).to receive(:warn)
+          service.find_or_retrieve(file_set, filepath: argument_filepath)
+        end
+        it "returns nil" do
+          expect(service.find_or_retrieve(file_set, filepath: argument_filepath)).to be_nil
+        end
+      end
+    end
+    context "when filepath provided" do
+      let(:argument_filepath) { '/tmp/existing_file.txt' }
+      context "when file present" do
+        before { allow(File).to receive(:exist?).with(argument_filepath).and_return(true) }
+        it "returns the filepath" do
+          expect(service.find_or_retrieve(file_set, filepath: argument_filepath)).to eq argument_filepath
+        end
+      end
+      context "when file absent" do
+        include_examples "find_or_retrieve examples", '/tmp/existing_file.txt'
+      end
+    end
+    context "when filepath not provided" do
+      include_examples "find_or_retrieve examples", nil
+    end
+  end
 end