Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/controllers/concerns/csv2db/controller_helpers.rb
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def enqueue_csv_import_and_redirect(klass, options = {}, &block)
end

def enqueue_csv_import(klass, options = {})
permitted_params = options.fetch(:params) do
permitted_params = options.fetch(:params) do
params.require(klass.model_name.param_key).permit(
:file,
*options[:extra_params]
Expand Down
64 changes: 64 additions & 0 deletions app/models/concerns/csv2db/active_storage_adapter.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
module Csv2db::ActiveStorageAdapter
require 'active_support/all'

extend ActiveSupport::Concern
FILE_TYPE = 'text/csv'.freeze
LINK_MAX_EXPIRY = 7.days.to_s.freeze

included do
has_one_attached Csv2db.config.file_attachment_name

validate :check_file_extension

alias_method :file_attachment, Csv2db.config.file_attachment_name
end

def file=(file)
# Override Dragonfly setter method
return unless file.present?

filename = file.original_filename

file_attachment.attach(
io: File.open(file),
filename: filename,
content_type: file.content_type
)

self.file_name = filename
end

def expiring_link(expires_in: LINK_MAX_EXPIRY)
return unless file_attachment.present?

set_current_host

file_attachment.service_url(expires_in: expires_in.to_i, disposition: 'attachment')
end

private

def set_current_host
return unless %i[test local].include?(Rails.application.config.active_storage.service)

ActiveStorage::Current.host = ReportGenerator.config.local_storage_host
end

def check_file_extension
# very basic check of file extension
errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file_attachment.blob.content_type == FILE_TYPE
end

def file_data
return @file_data if @file_data.present?

file_attachment.blob.open do |blob|
@file_data = str_to_utf8(blob.read)
end

byte_order_mark = Csv2db::Import::BYTE_ORDER_MARK
@file_data.sub!(byte_order_mark, '') if @file_data.starts_with?(byte_order_mark)

@file_data
end
end
27 changes: 27 additions & 0 deletions app/models/concerns/csv2db/dragonfly_adapter.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
module Csv2db::DragonflyAdapter
extend ActiveSupport::Concern
require 'dragonfly'

included do
extend Dragonfly::Model

dragonfly_accessor :file

validates :file, presence: true
validate :check_file_extension
end

private

def check_file_extension
# very basic check of file extension
errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file.ext == 'csv'
end

def file_data
file_data = str_to_utf8(file.data)
byte_order_mark = Csv2db::Import::BYTE_ORDER_MARK
file_data.sub!(byte_order_mark, '') if file_data.starts_with?(byte_order_mark)
file_data
end
end
32 changes: 13 additions & 19 deletions app/models/concerns/csv2db/import.rb
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,9 @@ def around_process(*args, &block)
end

included do
extend Dragonfly::Model
include Module.const_get("Csv2db::#{Csv2db.config.storage_adapter.camelize.constantize}Adapter")

validates :file, presence: true
validate :required_params_are_present
validate :check_file_extension

dragonfly_accessor :file

after_initialize :set_default_values, :set_required_params

Expand Down Expand Up @@ -128,10 +124,14 @@ def method_missing(method, *args, &block)
end
end

def respond_to_missing?(method, include_private = false)
method.to_s.start_with?('param_') || super
end

private

def check_file_contains_data
error(I18n.t('shared.file_processor.insufficient_rows')) unless file.data.present? && csv.count > 0
error(I18n.t('shared.file_processor.insufficient_rows')) unless csv.headers.present? && csv.count.positive?
stop if errors?
end

Expand Down Expand Up @@ -165,12 +165,6 @@ def csv
@csv ||= CSV.parse(file_data, headers: true)
end

def file_data
file_data = str_to_utf_8(file.data)
file_data.sub!(BYTE_ORDER_MARK, '') if file_data.starts_with?(BYTE_ORDER_MARK)
file_data
end

def required_params_are_present
return if @required_params.empty?

Expand All @@ -183,7 +177,7 @@ def required_params_are_present
end

def log(message, level = :info)
log_messages << { message: str_to_utf_8(message), level: level, time: Time.now }
log_messages << { message: str_to_utf8(message), level: level, time: Time.now }
end

def error(message)
Expand All @@ -203,16 +197,16 @@ def set_default_values
self.status ||= Status::PENDING
end

def str_to_utf_8(str)
CharlockHolmes::Converter.convert(str, str.detect_encoding[:encoding], 'UTF-8')
def str_to_utf8(str)
CharlockHolmes::Converter.convert(str, str_encoding(str), 'UTF-8')
end

def set_required_params
@required_params = []
def str_encoding(str)
str.detect_encoding[:encoding]
end

def check_file_extension
errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file.ext == 'csv'
def set_required_params
@required_params = []
end
end
end
5 changes: 4 additions & 1 deletion app/views/csv2db/_csv_import_table.html.haml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
%tr
%td= import.id
%td= import.created_at
%td= link_to import.file.name, import.file.url, target: '_blank'
- if Csv2db.config.storage_adapter.active_storage?
%td= link_to import.file_name, import.expiring_link
- else
%td= link_to import.file.name, import.file.url, target: '_blank'
%td
.badge{ class: "badge-upload-#{import.status}" }= import.status
%td
Expand Down
12 changes: 11 additions & 1 deletion lib/csv2db.rb
Original file line number Diff line number Diff line change
@@ -1,8 +1,18 @@
require 'csv2db/version'
require 'csv2db/config'

module Csv2db
class Error < StandardError; end
# Your code goes here...

class << self
def config
Config.instance
end

def configure
yield(config)
end
end
end

require 'csv2db/rails' if defined?(Rails)
22 changes: 22 additions & 0 deletions lib/csv2db/config.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
require 'singleton'

module Csv2db
class Config
include Singleton

attr_writer :storage_adapter, :local_storage_host, :file_attachment_name

def storage_adapter
@storage_adapter ||= :dragonfly
ActiveSupport::StringInquirer.new(@storage_adapter.to_s)
end

def local_storage_host
@local_storage_host ||= ''
end

def file_attachment_name
@file_attachment_name ||= :file_attachment
end
end
end
40 changes: 40 additions & 0 deletions spec/models/concerns/csv2db/import_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -104,4 +104,44 @@ class TestModel < ActiveRecord::Base
expect(subject.errors?).to be_truthy
end
end

context 'ActiveStorageAdapter' do
let(:file) do
Rack::Test::UploadedFile.new(Tempfile.new)
end

let(:attachment_spy) do
spy('file_attachment')
end

subject do
TestModel.new
end

before do
allow(TestModel).to receive(:has_one_attached)
allow(TestModel).to receive(:alias_method)
.with(:file_attachment, :file_attachment).and_return(nil)
TestModel.include(Csv2db::ActiveStorageAdapter)
allow(subject).to receive(:file_attachment).and_return(attachment_spy)
end

it 'calls correct attach methods' do
expect(file).to receive(:original_filename)
expect(file).to receive(:content_type)
expect(attachment_spy).to receive(:attach)

subject.file = file
end

it 'sets the file_name on the model' do
subject.file = file

expect(subject.file_name).to eq(file.original_filename)
end

it 'returns nil if no file passed' do
expect(subject.file = nil).to eq(nil)
end
end
end
2 changes: 2 additions & 0 deletions spec/spec_helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
require 'csv2db'
require_relative '../app/models/concerns/csv2db/import'
require_relative '../app/workers/csv2db/import_worker'
require_relative '../app/models/concerns/csv2db/dragonfly_adapter'
require_relative '../app/models/concerns/csv2db/active_storage_adapter'

ENV['RAILS_ENV'] ||= 'test'

Expand Down