Handle column headers with numeric suffixes

Author: laritakr

app/services/bulkrax/csv_validation_service/validator.rb

@@ -34,11 +34,14 @@ def initialize(csv_headers, valid_headers, field_metadata, mapping_manager, file
       end
 
       # Find required fields that are missing from the CSV
+      # Headers with numeric suffixes (_1, _2, etc.) are normalized before checking
+      # (e.g., 'title_1' satisfies the 'title' requirement)
       #
       # @return [Array<Hash>] Array of hashes with :model and :field keys
       def missing_required_fields
         @missing_required_fields ||= begin
-          csv_hdrs = @csv_headers.map { |h| @mapping_manager.mapped_to_key(h) }
+          # Map headers through field mappings and normalize suffixes
+          csv_hdrs = @csv_headers.map { |h| normalize_header(@mapping_manager.mapped_to_key(h)) }.uniq
 
           missing = []
           @field_metadata.each do |model, metadata|
@@ -52,10 +55,17 @@ def missing_required_fields
       end
 
       # Find headers in CSV that are not recognized as valid fields
+      # Headers with numeric suffixes (_1, _2, etc.) are normalized to their base form
+      # before comparison (e.g., 'creator_1' is treated as 'creator')
       #
       # @return [Array<String>] Array of unrecognized header names
       def unrecognized_headers
-        @unrecognized_headers ||= @csv_headers - @valid_headers
+        @unrecognized_headers ||= begin
+          @csv_headers.reject do |header|
+            normalized = normalize_header(header)
+            @valid_headers.include?(header) || @valid_headers.include?(normalized)
+          end
+        end
       end
 
       # Check if CSV is valid (no missing required fields and has headers)
@@ -71,6 +81,23 @@ def valid?
       def warnings?
         unrecognized_headers.any? || (@file_validator&.missing_files&.any? || false)
       end
+
+      private
+
+      # Normalize a header by stripping numeric suffixes
+      # Handles multi-valued fields where CSV columns like 'creator_1', 'creator_2'
+      # should be recognized as valid if 'creator' is a valid field
+      #
+      # @param header [String] The header to normalize
+      # @return [String] The normalized header (without numeric suffix)
+      #
+      # @example
+      #   normalize_header('creator_1')   # => 'creator'
+      #   normalize_header('title_2')     # => 'title'
+      #   normalize_header('source_identifier') # => 'source_identifier'
+      def normalize_header(header)
+        header.sub(/_\d+\z/, '')
+      end
     end
   end
 end

app/services/bulkrax/stepper_response_formatter.rb

@@ -84,7 +84,7 @@ def format
       # Build formatted response with messages structure
       {
         headers: @data[:headers],
-        missingRequired: @data[:missingRequired],
+        missingRequired: normalize_missing_required(@data[:missingRequired]),
         unrecognized: @data[:unrecognized],
         rowCount: @data[:rowCount],
         isValid: @data[:isValid],
@@ -112,6 +112,19 @@ def already_formatted?
         @data[:messages].key?(:validationStatus)
     end
 
+    # Normalize missingRequired to array of strings
+    # Converts [{model: 'Work', field: 'title'}] to ['title']
+    #
+    # @param missing_required [Array] Array of hashes or strings
+    # @return [Array<String>] Array of field names only
+    def normalize_missing_required(missing_required)
+      return [] unless missing_required
+
+      missing_required.map do |item|
+        item.is_a?(Hash) ? item[:field] : item
+      end
+    end
+
     # Build the messages structure with validationStatus and issues
     #
     # @return [Hash] Messages structure for frontend
@@ -173,14 +186,16 @@ def details_message(recognized)
     #
     # @return [Hash] Missing required fields issue structure
     def missing_required_issue
+      normalized = normalize_missing_required(@data[:missingRequired])
+
       {
         type: 'missing_required_fields',
         severity: 'error',
         icon: 'fa-times-circle',
         title: 'Missing Required Fields',
-        count: @data[:missingRequired].length,
+        count: normalized.length,
         description: 'These required columns must be added to your CSV:',
-        items: @data[:missingRequired].map { |field| { field: field, message: 'add this column to your CSV' } },
+        items: normalized.map { |field| { field: field, message: 'add this column to your CSV' } },
         defaultOpen: false
       }
     end

spec/services/bulkrax/csv_validation_service/validator_spec.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe Bulkrax::CsvValidationService::Validator do
+  let(:mapping_manager) { instance_double(Bulkrax::CsvValidationService::MappingManager) }
+  let(:file_validator) { nil }
+
+  let(:field_metadata) do
+    {
+      'Work' => {
+        properties: %w[title creator description],
+        required_terms: %w[title source_identifier],
+        controlled_vocab_terms: []
+      },
+      'Collection' => {
+        properties: %w[title description],
+        required_terms: %w[title],
+        controlled_vocab_terms: []
+      }
+    }
+  end
+
+  let(:valid_headers) do
+    %w[model source_identifier title creator description parent file]
+  end
+
+  describe '#unrecognized_headers' do
+    context 'with standard headers' do
+      it 'returns empty array when all headers are recognized' do
+        csv_headers = %w[title creator description]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to be_empty
+      end
+
+      it 'identifies unrecognized headers' do
+        csv_headers = %w[title creator invalid_field another_bad_one]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to contain_exactly('invalid_field', 'another_bad_one')
+      end
+    end
+
+    context 'with numeric suffixes' do
+      it 'recognizes headers with _1 suffix as valid' do
+        csv_headers = %w[title_1 creator_1 description_1]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to be_empty
+      end
+
+      it 'recognizes headers with _2 suffix as valid' do
+        csv_headers = %w[title_2 creator_2]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to be_empty
+      end
+
+      it 'recognizes headers with multi-digit suffixes' do
+        csv_headers = %w[title_10 creator_99 description_123]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to be_empty
+      end
+
+      it 'identifies unrecognized headers even with numeric suffixes' do
+        csv_headers = %w[title_1 invalid_field_1 creator_2 another_bad_2]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to contain_exactly('invalid_field_1', 'another_bad_2')
+      end
+
+      it 'handles mix of suffixed and non-suffixed headers' do
+        csv_headers = %w[title title_1 title_2 creator creator_1 description]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to be_empty
+      end
+
+      it 'does not strip non-numeric suffixes' do
+        csv_headers = %w[title_abc creator_xyz]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to contain_exactly('title_abc', 'creator_xyz')
+      end
+
+      it 'handles underscores in field names correctly' do
+        csv_headers = %w[source_identifier source_identifier_1 source_identifier_2]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.unrecognized_headers).to be_empty
+      end
+    end
+  end
+
+  describe '#missing_required_fields' do
+    before do
+      allow(mapping_manager).to receive(:mapped_to_key) { |h| h }
+    end
+
+    it 'returns empty array when all required fields are present' do
+      csv_headers = %w[title source_identifier model]
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      expect(validator.missing_required_fields).to be_empty
+    end
+
+    it 'identifies missing required fields' do
+      csv_headers = %w[creator description]
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      missing = validator.missing_required_fields
+      expect(missing).to include({ model: 'Work', field: 'title' })
+      expect(missing).to include({ model: 'Work', field: 'source_identifier' })
+      expect(missing).to include({ model: 'Collection', field: 'title' })
+    end
+
+    it 'returns unique missing fields across models' do
+      csv_headers = %w[description]
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      missing = validator.missing_required_fields
+      title_missing = missing.select { |m| m[:field] == 'title' }
+
+      # Both Work and Collection require title, so we should see 2 entries
+      expect(title_missing.length).to eq(2)
+    end
+
+    it 'works with mapped column names' do
+      allow(mapping_manager).to receive(:mapped_to_key) do |header|
+        { 'work_title' => 'title', 'identifier' => 'source_identifier' }[header] || header
+      end
+
+      csv_headers = %w[work_title identifier]
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      expect(validator.missing_required_fields).to be_empty
+    end
+
+    context 'with numeric suffixes' do
+      it 'recognizes title_1 as satisfying title requirement' do
+        csv_headers = %w[title_1 source_identifier]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.missing_required_fields).to be_empty
+      end
+
+      it 'recognizes multiple suffixed headers as satisfying requirement' do
+        csv_headers = %w[title_1 title_2 source_identifier_1]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.missing_required_fields).to be_empty
+      end
+
+      it 'still identifies missing required fields when suffixed headers do not match' do
+        csv_headers = %w[creator_1 description_2]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        missing = validator.missing_required_fields
+        expect(missing).to include({ model: 'Work', field: 'title' })
+        expect(missing).to include({ model: 'Work', field: 'source_identifier' })
+        expect(missing).to include({ model: 'Collection', field: 'title' })
+      end
+
+      it 'works with mix of suffixed and non-suffixed required fields' do
+        csv_headers = %w[title source_identifier_1]
+        validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+        expect(validator.missing_required_fields).to be_empty
+      end
+    end
+  end
+
+  describe '#valid?' do
+    before do
+      allow(mapping_manager).to receive(:mapped_to_key) { |h| h }
+    end
+
+    it 'returns true when all required fields are present' do
+      csv_headers = %w[title source_identifier model]
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      expect(validator).to be_valid
+    end
+
+    it 'returns false when required fields are missing' do
+      csv_headers = %w[description creator]
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      expect(validator).not_to be_valid
+    end
+
+    it 'returns false when CSV has no headers' do
+      csv_headers = []
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      expect(validator).not_to be_valid
+    end
+  end
+
+  describe '#warnings?' do
+    before do
+      allow(mapping_manager).to receive(:mapped_to_key) { |h| h }
+    end
+
+    it 'returns true when there are unrecognized headers' do
+      csv_headers = %w[title creator invalid_field]
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      expect(validator.warnings?).to be true
+    end
+
+    it 'returns false when all headers are recognized' do
+      csv_headers = %w[title creator description]
+      validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager)
+
+      expect(validator.warnings?).to be false
+    end
+
+    context 'with file validator' do
+      let(:file_validator) do
+        instance_double(
+          Bulkrax::CsvValidationService::FileValidator,
+          missing_files: missing_files
+        )
+      end
+
+      context 'when files are missing' do
+        let(:missing_files) { ['image1.jpg', 'doc.pdf'] }
+
+        it 'returns true' do
+          csv_headers = %w[title creator]
+          validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager, file_validator)
+
+          expect(validator.warnings?).to be true
+        end
+      end
+
+      context 'when no files are missing' do
+        let(:missing_files) { [] }
+
+        it 'returns false when headers are also valid' do
+          csv_headers = %w[title creator]
+          validator = described_class.new(csv_headers, valid_headers, field_metadata, mapping_manager, file_validator)
+
+          expect(validator.warnings?).to be false
+        end
+      end
+    end
+  end
+end