Header validation changes for Tind Spreadsheet validator

Author: davezuckerman

app/lib/tind_spread/spread_tool.rb

@@ -75,24 +75,28 @@ def find_largest(ffts)
       highest
     end
 
-    # rubocop:disable Lint/UselessAssignment
+    # rubocop:disable Lint/UselessAssignment, Metrics/AbcSize
     def get_files(file_pattern)
       dir = "#{Rails.application.config.tind_data_root_dir}/#{@directory.delete_prefix('/')}"
-      if file_pattern.nil?
+      if file_pattern.nil? || !file_pattern.respond_to?(:gsub)
         matches = []
       else
         file_match = file_pattern.gsub(/\.tif/i, '')
         matches = File.directory?("#{dir}/#{file_pattern}") ? Dir.glob("#{dir}/*/#{file_match}*") : Dir.glob("#{dir}/#{file_match}*")
         matches.map! { |i| i.gsub(Rails.application.config.tind_data_root_dir, 'https://digitalassets.lib.berkeley.edu') }
       end
     end
-    # rubocop:enable Lint/UselessAssignment
+    # rubocop:enable Lint/UselessAssignment, Metrics/AbcSize
 
     def get_ffts(all_rows)
       ffts = []
       all_rows.each do |row_data|
         file_pattern = get_filename(row_data)
-        ffts << urls_to_fft(get_files(file_pattern))
+        ffts << if file_pattern.is_a?(String) && file_pattern.present?
+                  urls_to_fft(get_files(file_pattern))
+                else
+                  {}
+                end
       end
       ffts
     end

app/lib/tind_spread/tind_validation.rb

@@ -7,9 +7,16 @@ module TindValidation
     # validates the header row
     # should be 3 digits for field, 2 for indicator (can be _, number), one digit or number for subfield
     # optionally can have a ('-' followed by a number). This is used to group columns into similar fields
-    # the header row can also be just "Filename" or "FFT". The program will create the proper fields for those
+    # the header row can also be just "Filename" or "001". The program will create the proper fields for those
     def self.valid_header?(str)
-      str.match?(/\d{3}[_|\d]{2}[a-zA-Z0-9](-\d+)?$/) || str.match?(/Filename|FFT/i)
+      valid_patterns = [
+        /^\d+:001/,
+        /^\d+:\d{3}[_|\d]{2}[a-zA-Z0-9](-\d+)?$/,
+        /^\d+:Filename$/i,
+        /FFT/
+      ]
+
+      valid_patterns.any? { |pattern| pattern.match?(str) }
     end
 
     # runs a set of validations against a single row.

spec/lib/tind_spread/spread_tool_spec.rb

@@ -85,4 +85,22 @@
       expect(spread_tool.spread_to_hash(header)).to eq(expected_result)
     end
   end
+
+  describe '#get_ffts' do
+    it 'skips get_files when the filename is blank' do
+      row_data = [{ '0:Filename' => nil, '1:Header2' => 'Data2' }]
+
+      expect(spread_tool).not_to receive(:get_files)
+
+      expect(spread_tool.get_ffts(row_data)).to eq([{}])
+    end
+
+    it 'skips get_files when the filename is a hash' do
+      row_data = [{ '0:Filename' => { name: 'Data1' }, '1:Header2' => 'Data2' }]
+
+      expect(spread_tool).not_to receive(:get_files)
+
+      expect(spread_tool.get_ffts(row_data)).to eq([{}])
+    end
+  end
 end

spec/lib/tind_spread/tind_batch_spec.rb

@@ -15,7 +15,7 @@
   let(:args) { { directory:, '982__a': 'test' } }
   let(:tind_batch) { described_class.new(args, xlsx, extension, email) }
   let(:spread_tool) { instance_double(TindSpread::SpreadTool) }
-  let(:all_rows) { [{ '001__a' => 'Data1', '245__a' => 'Data2' }, { '001__a' => 'Data3', '245__a' => 'Data4' }] }
+  let(:all_rows) { [{ '1:001__a' => 'Data1', '1:245__a' => 'Data2' }, { '1:001__a' => 'Data3', '1:245__a' => 'Data4' }] }
 
   before do
     allow(TindSpread::SpreadTool).to receive(:new).with(xlsx, extension, directory).and_return(spread_tool)
@@ -76,7 +76,7 @@
 
   describe '#validate_header_row' do
     it 'returns an empty array for valid headers' do
-      headers = %w[001__a 245__a 500__3]
+      headers = %w[1:001__a 1:245__a 1:500__3]
       errors = tind_batch.validate_header_row(headers)
       expect(errors).to be_empty
     end
@@ -89,7 +89,7 @@
     end
 
     it 'returns errors only for invalid headers in a mixed list' do
-      headers = %w[001__a InvalidHeader 245__a]
+      headers = %w[1:001__a InvalidHeader 1:245__a]
       errors = tind_batch.validate_header_row(headers)
       expect(errors).to include('Invalid header name: InvalidHeader')
       expect(errors.length).to eq(1)

spec/lib/tind_spread/tind_validation_spec.rb

@@ -111,16 +111,17 @@
 
   describe '.valid_header?' do
     it 'returns true for a valid header with standard format' do
-      expect(described_class.valid_header?('001__a')).to be true
-      expect(described_class.valid_header?('245__a')).to be true
-      expect(described_class.valid_header?('Filename')).to be true
-      expect(described_class.valid_header?('100_1a')).to be true
+      expect(described_class.valid_header?('1:001')).to be true
+      expect(described_class.valid_header?('1:001__a')).to be true
+      expect(described_class.valid_header?('1:245__a')).to be true
+      expect(described_class.valid_header?('1:Filename')).to be true
+      expect(described_class.valid_header?('1:100_1a')).to be true
     end
 
     it 'returns true for a valid header with suffix format' do
-      expect(described_class.valid_header?('001__a-1')).to be true
-      expect(described_class.valid_header?('245__a-2')).to be true
-      expect(described_class.valid_header?('500__3-5')).to be true
+      expect(described_class.valid_header?('1:001__a-1')).to be true
+      expect(described_class.valid_header?('1:245__a-2')).to be true
+      expect(described_class.valid_header?('1:500__3-5')).to be true
     end
 
     it 'returns false for invalid headers' do