diff --git a/.gitignore b/.gitignore index 39cfdaf840..6629b0283e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ Gemfile.ci.lock public/avatars/**/* public/assets +public/importers/ tmp Design @@ -37,3 +38,6 @@ Design .passenger .vagrant + +docker-compose.override.yml +.local/ diff --git a/Dockerfile b/Dockerfile index b84d416d1c..e55373aedf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,8 @@ RUN apt-get update && \ apt-get autoremove -y && \ cp config/database.postgres.docker.yml config/database.yml && \ gem install bundler && \ - bundle install --deployment && \ + bundle config set deployment 'true' && \ + bundle install && \ bundle exec rails assets:precompile CMD ["bundle","exec","rails","s"] diff --git a/Gemfile b/Gemfile index 99a6106d07..d2e4040e13 100644 --- a/Gemfile +++ b/Gemfile @@ -100,3 +100,8 @@ gem "devise-encryptable" gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby] gem 'activejob', '~> 5.2.0' gem 'ransack_ui', path: 'vendor/gems/ransack_ui-1.3.4' # Vendored until our fix is merged and released +gem 'spreadsheet' + +gem "roo", "~> 2.8" + +gem "roo-xls", "~> 1.2" diff --git a/Gemfile.lock b/Gemfile.lock index e22d3ee844..c71acde36c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -176,6 +176,7 @@ GEM jquery-ui-rails (6.0.1) railties (>= 3.2.16) libv8 (3.16.14.19) + libv8 (3.16.14.19-x86_64-linux) listen (3.2.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) @@ -302,6 +303,13 @@ GEM railties (>= 4.2.0, < 6.0) responds_to_parent (2.0.0) actionpack (>= 3.2.22, < 6.0) + roo (2.8.3) + nokogiri (~> 1) + rubyzip (>= 1.3.0, < 3.0.0) + roo-xls (1.2.0) + nokogiri + roo (>= 2.0.0, < 3) + spreadsheet (> 0.9.0) rspec (3.9.0) rspec-core (~> 3.9.0) rspec-expectations (~> 3.9.0) @@ -334,6 +342,7 @@ GEM rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 1.7) + ruby-ole (1.2.12.2) ruby-progressbar (1.10.1) rubyzip (2.3.0) sass (3.7.4) @@ -359,6 +368,8 @@ GEM sixarm_ruby_unaccent (1.2.0) sort_alphabetical (1.1.0) unicode_utils (>= 1.2.2) + spreadsheet (1.2.6) + ruby-ole (>= 1.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -403,6 +414,7 @@ GEM PLATFORMS ruby + x86_64-linux DEPENDENCIES activejob (~> 5.2.0) @@ -462,6 +474,8 @@ DEPENDENCIES rb-inotify responders (~> 2.0) responds_to_parent + roo (~> 2.8) + roo-xls (~> 1.2) rspec-activemodel-mocks rspec-rails rubocop (~> 0.76.0) @@ -470,6 +484,7 @@ DEPENDENCIES select2-rails selenium-webdriver simple_form + spreadsheet sprockets-rails (>= 3.0.0) sqlite3 (~> 1.3.13) therubyracer @@ -482,4 +497,4 @@ DEPENDENCIES zeus BUNDLED WITH - 2.1.4 + 2.2.2 diff --git a/app/controllers/importers_controller.rb b/app/controllers/importers_controller.rb new file mode 100644 index 0000000000..feb4481771 --- /dev/null +++ b/app/controllers/importers_controller.rb @@ -0,0 +1,186 @@ +# frozen_string_literal: true + +# Copyright (c) 2008-2013 Michael Dvorkin and contributors. +# +# Fat Free CRM is freely distributable under the terms of MIT license. +# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php +#------------------------------------------------------------------------------ + +require 'json' + +class ImportersController < ApplicationController + # get /importers/new AJAX + #---------------------------------------------------------------------------- + def new + @importer = Importer.new + @importer.entity_type = params[:entity_type] + @importer.entity_id = params[:entity_id] if params[:entity_id] + respond_with(@importer) + end + + # post /importers/create + #---------------------------------------------------------------------------- + def create + errors = false + if params[:importer] + @importer = Importer.create(importer_params) + if @importer.valid? + @importer.save + else + errors = @importer.errors.full_messages + end + end + + respond_to do |format| + if errors + format.html { render "create", locals: { errors: errors } } + else + format.html { redirect_to form_map_columns_importer_path(@importer) } + end + end + end + + # get /importers/:id/map + #---------------------------------------------------------------------------- + def form_map_columns + @importer = Importer.find(params[:id]) + columns = FatFreeCRM::ImportHandle.get_columns(@importer.attachment.path) + + attributes = [] + attributes_extra = [] + + object = @importer.entity_class + attrs = object.attribute_names - ['id'] + + attrs.each do |attr| + attributes.push( + name: attr, + required: object.validators_on(attr).any? { |v| v.is_a? ActiveModel::Validations::PresenceValidator } + ) + end + + if @importer.entity_type == 'lead' + attrs = Address.attribute_names - %w[id created_at updated_at deleted_at address_type addressable_type addressable_id] + + attrs.each do |attr| + attributes_extra.push( + name: attr, + required: Address.validators_on(attr).any? { |v| v.is_a? ActiveModel::Validations::PresenceValidator } + ) + end + end + + respond_to do |format| + format.html { render "form_map_columns", locals: { columns: columns, attributes: attributes, attributes_extra: attributes_extra } } + end + end + + # post /importers/:id/map + #---------------------------------------------------------------------------- + def map_columns + @importer = Importer.find(params[:id]) + @importer.status = :map + map = params[:map] + @importer.map = map.to_json + @importer.save + @importer = FatFreeCRM::ImportHandle.process(@importer) + + respond_to do |format| + format.html { render "map_columns" } + end + end + + # # get /campaigns/import AJAX + # #---------------------------------------------------------------------------- + # def import + # @importer = Importer.new + # @importer.entity_type = 'Campaign' + # respond_with(@importer) + # end + # + # # patch /campaigns/import AJAX + # #---------------------------------------------------------------------------- + # def import_upload + # @error = false + # @result = { + # items: [], + # errors: [] + # } + # + # if params[:importer] + # @importer = Importer.create(import_params) + # if @importer.valid? + # @importer.save + # @result = FatFreeCRM::ImportHandle.process(@importer) + # else + # puts @importer.errors.full_messages + # @result[:errors].push(@importer.errors.full_messages) + # @error = true + # end + # end + # respond_with(@error,@result) + # end + # + # + # # get /campaigns/%id/import AJAX + # #---------------------------------------------------------------------------- + # def import_leads + # @importer = Importer.new + # @importer.entity_type = 'Lead' + # respond_with(@importer) + # end + # + # # patch /campaigns/import AJAX + # #---------------------------------------------------------------------------- + # def uploads_import_leads + # @error = false + # @result = { + # items: [], + # errors: [] + # } + # + # if params[:importer] + # @importer = Importer.create(import_params) + # if @importer.valid? + # @importer.save + # @colummns = FatFreeCRM::ImportHandle.get_columns(@importer.attachment.path) + # else + # puts @importer.errors.full_messages + # @result[:errors].push(@importer.errors.full_messages) + # @error = true + # end + # end + # respond_with(@colummns) do |format| + # format.js { render :uploads_import_leads } + # end + # end + # + # post /importers/create + #---------------------------------------------------------------------------- + # def create + # @error = false + # @result = { + # items: [], + # errors: [] + # } + # + # if params[:importer] + # @importer = Importer.create(import_params) + # if @importer.valid? + # @importer.save + # @result = FatFreeCRM::ImportHandle.process(@importer) + # else + # puts @importer.errors.full_messages + # @result[:errors].push(@importer.errors.full_messages) + # @error = true + # end + # end + # respond_with(@error,@result) + # end + + private + + def importer_params + params.require(:importer).permit(:attachment, :entity_type, :entity_id) + end +end diff --git a/app/models/files/imported_file.rb b/app/models/files/imported_file.rb new file mode 100644 index 0000000000..f0c019dd8e --- /dev/null +++ b/app/models/files/imported_file.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Copyright (c) 2008-2013 Michael Dvorkin and contributors. +# +# Fat Free CRM is freely distributable under the terms of MIT license. +# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php +#------------------------------------------------------------------------------ +# == Schema Information +# +# Table name: imported_files +# +# id :integer not null, primary key +# filename :string(64) default(""), not null +# md5sum :string(32) default(""), not null +# + +class ImportedFile < ActiveRecord::Base + before_validation :generate_md5sum + + validate :filetype + + validates :filename, presence: true + validates :md5sum, presence: true + validates :md5sum, uniqueness: { message: "file already imported" } + + def generate_md5sum + self.md5sum = Digest::MD5.hexdigest File.open(filename).read unless filename.empty? + rescue StandardError + "" + end + + private + + def filetype + valid = begin + File.open(filename).type_from_file_command == "application/vnd.ms-excel" + rescue StandardError + "" + end + errors.add(:filename, "no such file") if valid == "" + errors.add(:filename, "invalid filetype") unless valid + end + + ActiveSupport.run_load_hooks(:fat_free_crm_imported_file, self) +end diff --git a/app/models/files/importer.rb b/app/models/files/importer.rb new file mode 100644 index 0000000000..1c04f94dea --- /dev/null +++ b/app/models/files/importer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: importers +# +# id :integer not null, primary key +# entity_type :string +# entity_id :integer +# attachment_file_size :integer +# attachment_file_name :string(255) +# attachment_content_type :string(255) +# status :string(255) +# created_at :datetime +# updated_at :datetime +# +require 'json' +class Importer < ActiveRecord::Base + attribute :entity_attrs + + has_attached_file :attachment, path: ":rails_root/public/importers/:id/:filename" + + validates_attachment :attachment, presence: true + + validates_attachment_content_type :attachment, + content_type: %w[text/xml application/xml application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet application/x-ole-storage], + message: 'Only EXCEL files are allowed.' + validates_attachment_file_name :attachment, matches: [/\.xls/, /\.xlsx?$/] + + def messages + JSON.parse(messages) + end + + def entity_class + entity_type.capitalize.constantize + end + + ActiveSupport.run_load_hooks(:fat_free_crm_importer, self) +end diff --git a/app/views/campaigns/_list_title_bar.html.haml b/app/views/campaigns/_list_title_bar.html.haml new file mode 100644 index 0000000000..3de3f01ab6 --- /dev/null +++ b/app/views/campaigns/_list_title_bar.html.haml @@ -0,0 +1,14 @@ +- model_name = controller_name.singularize +- model_klass = model_name.camelcase.constantize + +.title_tools + #buttons + = view_buttons + .create_asset + = link_to_inline("create_#{model_name}".to_sym, send("new_#{model_name}_path"), text: t("create_#{model_name}".to_sym)) + .create_asset + = link_to_inline(:new_importer, new_importer_path(:campaign), text: t(:import_campaigns)) + +.title + %span{id: "create_#{model_name}_title"} #{t controller_name.to_sym} + = image_tag("loading.gif", size: :thumb, id: "loading", style: "display: none;") diff --git a/app/views/campaigns/_title_bar.html.haml b/app/views/campaigns/_title_bar.html.haml index 3933786f88..1a2013aeed 100644 --- a/app/views/campaigns/_title_bar.html.haml +++ b/app/views/campaigns/_title_bar.html.haml @@ -1,7 +1,9 @@ #confirm{ hidden } .title_tools#menu = link_to_inline(:edit_campaign, edit_campaign_path(@campaign), text: t(:edit)) + " | " - = link_to_function(t(:delete) + '?', confirm_delete(@campaign)) + = link_to_function(t(:delete) + '?', confirm_delete(@campaign)) + " | " + = link_to_inline(:new_importer, new_importer_path(:lead,@campaign), text: t(:import_leads)) + .title_tools#buttons = view_buttons .title#edit_campaign_title diff --git a/app/views/campaigns/index.html.haml b/app/views/campaigns/index.html.haml index 9f1c71384a..5318765377 100644 --- a/app/views/campaigns/index.html.haml +++ b/app/views/campaigns/index.html.haml @@ -1,8 +1,9 @@ = styles_for :campaign -= render 'entities/title_bar' += render 'list_title_bar' .remote#create_campaign{ hidden } +.remote#new_importer{ hidden } = render 'search' diff --git a/app/views/campaigns/show.html.haml b/app/views/campaigns/show.html.haml index 10f3396cd7..9a26c92c3d 100755 --- a/app/views/campaigns/show.html.haml +++ b/app/views/campaigns/show.html.haml @@ -9,6 +9,8 @@ = render 'campaigns/title_bar', campaign: @campaign = render "comments/new", commentable: @campaign + .remote#new_importer{ hidden } + = render partial: "shared/timeline", collection: @timeline = hook(:show_campaign_bottom, self, {entity: @campaign}) do diff --git a/app/views/importers/_new.html.haml b/app/views/importers/_new.html.haml new file mode 100644 index 0000000000..0f36895ea4 --- /dev/null +++ b/app/views/importers/_new.html.haml @@ -0,0 +1,13 @@ += form_for(@importer, url: create_importer_path, html:{ multipart: true}) do |f| + = f.hidden_field :entity_type + - if @importer.entity_id + = f.hidden_field :entity_id + %p + %small #{t :importer_description} + + %div #{t :xls_file} + = fields_for(Importer) do |a| + %div= a.file_field :attachment + + .buttonbar + = f.submit t(:upload_file) \ No newline at end of file diff --git a/app/views/importers/create.html.haml b/app/views/importers/create.html.haml new file mode 100644 index 0000000000..ab6924e17a --- /dev/null +++ b/app/views/importers/create.html.haml @@ -0,0 +1,5 @@ +- if errors + - errors.each do |error| + .flash_error #{ error } +- else + #{ render(partial: "map_columns" ) } \ No newline at end of file diff --git a/app/views/importers/form_map_columns.html.haml b/app/views/importers/form_map_columns.html.haml new file mode 100644 index 0000000000..6f41028a9f --- /dev/null +++ b/app/views/importers/form_map_columns.html.haml @@ -0,0 +1,45 @@ += form_for(@importer, url: map_columns_importer_path(@importer), html: { method: :post }) do |f| + = f.hidden_field :id + + %p #{t :map_columns_description} + + %div + %table + - attributes.each do |attr| + %tr + %td + .label + #{ attr[:name] } + - if attr[:required] + %span.warn * + %td + = select_tag "map[#{ attr[:name] }]", + options_for_select(columns), + class: 'select2', + include_blank: true, + data: {placeholder: t(:select_blank)}, + required: attr[:required], + id: "map_#{ attr[:name] }" + + - if attributes_extra.length + %div.subtitle + #{t :address} + %table + - attributes_extra.each do |attr| + %tr + %td + .label + #{ attr[:name] } + - if attr[:required] + %span.warn * + %td + = select_tag "map[business_address_attributes][#{ attr[:name] }]", + options_for_select(columns), + class: 'select2', + include_blank: true, + data: {placeholder: t(:select_blank)}, + required: attr[:required], + id: "map_#{ attr[:name] }" + + .buttonbar + = f.submit t(:save) diff --git a/app/views/importers/map_columns.html.haml b/app/views/importers/map_columns.html.haml new file mode 100644 index 0000000000..78e1bc2c80 --- /dev/null +++ b/app/views/importers/map_columns.html.haml @@ -0,0 +1,4 @@ +%div #{t :importer_status_label } #{ @importer.status } +- @importer.messages.each do |message| + %div #{message} + diff --git a/app/views/importers/new.js.haml b/app/views/importers/new.js.haml new file mode 100644 index 0000000000..f6e50ef2e0 --- /dev/null +++ b/app/views/importers/new.js.haml @@ -0,0 +1,6 @@ +- _id = "new_importer" + +crm.flick('empty', 'toggle'); +crm.flip_form('#{_id}'); +$('##{_id}').html('#{ j render(partial: "new" ) }'); +crm.set_title('#{_id}', '#{ j t(_id) }'); diff --git a/config/initializers/file.rb b/config/initializers/file.rb new file mode 100644 index 0000000000..05a84dd024 --- /dev/null +++ b/config/initializers/file.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Copyright (c) 2008-2013 Michael Dvorkin and contributors. +# +# Fat Free CRM is freely distributable under the terms of MIT license. +# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php +#------------------------------------------------------------------------------ +File.class_eval do + def type_from_file_command + type = (begin + original_filename.match(/\.(\w+)$/)[1] + rescue StandardError + "octet-stream" + end).downcase + mime_type = begin + `file -b --mime-type #{path}`.split(':').last.strip + rescue StandardError + "application/x-#{type}" + end + mime_type = "application/x-#{type}" if mime_type.match(/\(.*?\)/) + mime_type + end +end diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb new file mode 100644 index 0000000000..e595b45f59 --- /dev/null +++ b/config/initializers/paperclip.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Paperclip + class MediaTypeSpoofDetector + def spoofed? + false + end + end +end diff --git a/config/locales/fat_free_crm.en-US.yml b/config/locales/fat_free_crm.en-US.yml index c2484983dc..38cb857c00 100644 --- a/config/locales/fat_free_crm.en-US.yml +++ b/config/locales/fat_free_crm.en-US.yml @@ -937,3 +937,16 @@ en-US: from_to: From %{from} to %{to} from_only: From %{from} to_only: Until %{to} + + + # Import from excel + #---------------------------------------------------------------------------- + import_campaigns: Import Campaigns + import_leads: Import Leads + import_leads_help: Todo help for import leads from excel + xls_file: EXCEL File + upload_file: Upload file + save: Save + importer_description: Upload excel file to import its content to the databases. Only valid excel files are allowed. + map_columns_description: Link the excel column that corresponds to each attribute or leave it blank. The elements with * must have a value in the column that is selected. + importer_status_label: Status \ No newline at end of file diff --git a/config/locales/fat_free_crm.es.yml b/config/locales/fat_free_crm.es.yml index cd3f805867..9a45704039 100644 --- a/config/locales/fat_free_crm.es.yml +++ b/config/locales/fat_free_crm.es.yml @@ -916,3 +916,15 @@ es: from_to: De %{from} a %{to} from_only: De %{from} to_only: Antes de %{to} + + # Import from excel + #---------------------------------------------------------------------------- + import_campaigns: Importar CampaƱas + import_leads: Importar clientes potenciales + import_leads_help: Todo help for import leads from excel + xls_file: Fichero EXCEL + upload_file: Subir fichero + save: Guardar + importer_description: Subir fichero excel para importar su contenido a las base de datos. Solo se admiten fichero excel validos. + map_columns_description: Enlazar la columna del excel que le corresponde a cada atributo o dejar en blanco. Los elementos con * tienen que tener valor en la columna que se seleccione. + importer_status_label: Estado diff --git a/config/routes.rb b/config/routes.rb index 1efaac95b3..9071fa58ed 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -209,4 +209,9 @@ resources :settings, only: :index resources :plugins, only: :index end + + get 'importers/new/:entity_type(/:entity_id)' => 'importers#new', as: :new_importer + post 'importers' => 'importers#create', as: :create_importer + get 'importers/:id/map' => 'importers#form_map_columns', as: :form_map_columns_importer + post 'importers/:id/map' => 'importers#map_columns', as: :map_columns_importer end diff --git a/db/migrate/20201103150431_create_imported_files.rb b/db/migrate/20201103150431_create_imported_files.rb new file mode 100644 index 0000000000..25af1acb8a --- /dev/null +++ b/db/migrate/20201103150431_create_imported_files.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class CreateImportedFiles < ActiveRecord::Migration[4.2] + def self.up + create_table :imported_files, force: true do |t| + t.string :filename, limit: 64, null: false, default: "" + t.string :md5sum, limit: 32, null: false, default: "" + + t.timestamps + end + end + + def self.down + drop_table :imported_files + end +end diff --git a/db/migrate/20201217030615_create_importers.rb b/db/migrate/20201217030615_create_importers.rb new file mode 100644 index 0000000000..e77f7b7777 --- /dev/null +++ b/db/migrate/20201217030615_create_importers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class CreateImporters < ActiveRecord::Migration[4.2] + def self.up + create_table :importers do |t| + t.integer :attachment_file_size # Uploaded file size + t.string :attachment_file_name, null: false # Uploaded full file name + t.string :attachment_content_type # MIME content type + t.string :entity_type, null: false # led, campaign + t.string :entity_id # led, campaign + t.string :status, null: false, default: :new # new, map , imported , error + t.text :map + t.text :messages + t.timestamps + end + end + + def self.down + drop_table :importers + end +end diff --git a/db/schema.rb b/db/schema.rb index 9fff439ce9..ce7a36775c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,9 +10,9 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_01_07_082701) do +ActiveRecord::Schema.define(version: 2020_12_17_030615) do - create_table "account_contacts", force: :cascade do |t| + create_table "account_contacts", id: :serial, force: :cascade do |t| t.integer "account_id" t.integer "contact_id" t.datetime "deleted_at" @@ -21,7 +21,7 @@ t.index ["account_id", "contact_id"], name: "index_account_contacts_on_account_id_and_contact_id" end - create_table "account_opportunities", force: :cascade do |t| + create_table "account_opportunities", id: :serial, force: :cascade do |t| t.integer "account_id" t.integer "opportunity_id" t.datetime "deleted_at" @@ -30,7 +30,7 @@ t.index ["account_id", "opportunity_id"], name: "index_account_opportunities_on_account_id_and_opportunity_id" end - create_table "accounts", force: :cascade do |t| + create_table "accounts", id: :serial, force: :cascade do |t| t.integer "user_id" t.integer "assigned_to" t.string "name", limit: 64, default: "", null: false @@ -53,7 +53,7 @@ t.index ["user_id", "name", "deleted_at"], name: "index_accounts_on_user_id_and_name_and_deleted_at", unique: true end - create_table "activities", force: :cascade do |t| + create_table "activities", id: :serial, force: :cascade do |t| t.integer "user_id" t.string "subject_type" t.integer "subject_id" @@ -66,7 +66,7 @@ t.index ["user_id"], name: "index_activities_on_user_id" end - create_table "addresses", force: :cascade do |t| + create_table "addresses", id: :serial, force: :cascade do |t| t.string "street1" t.string "street2" t.string "city", limit: 64 @@ -83,7 +83,7 @@ t.index ["addressable_id", "addressable_type"], name: "index_addresses_on_addressable_id_and_addressable_type" end - create_table "avatars", force: :cascade do |t| + create_table "avatars", id: :serial, force: :cascade do |t| t.integer "user_id" t.string "entity_type" t.integer "entity_id" @@ -94,7 +94,7 @@ t.datetime "updated_at" end - create_table "campaigns", force: :cascade do |t| + create_table "campaigns", id: :serial, force: :cascade do |t| t.integer "user_id" t.integer "assigned_to" t.string "name", limit: 64, default: "", null: false @@ -119,7 +119,7 @@ t.index ["user_id", "name", "deleted_at"], name: "index_campaigns_on_user_id_and_name_and_deleted_at", unique: true end - create_table "comments", force: :cascade do |t| + create_table "comments", id: :serial, force: :cascade do |t| t.integer "user_id" t.string "commentable_type" t.integer "commentable_id" @@ -131,7 +131,7 @@ t.string "state", limit: 16, default: "Expanded", null: false end - create_table "contact_opportunities", force: :cascade do |t| + create_table "contact_opportunities", id: :serial, force: :cascade do |t| t.integer "contact_id" t.integer "opportunity_id" t.string "role", limit: 32 @@ -141,7 +141,7 @@ t.index ["contact_id", "opportunity_id"], name: "index_contact_opportunities_on_contact_id_and_opportunity_id" end - create_table "contacts", force: :cascade do |t| + create_table "contacts", id: :serial, force: :cascade do |t| t.integer "user_id" t.integer "lead_id" t.integer "assigned_to" @@ -173,7 +173,7 @@ t.index ["user_id", "last_name", "deleted_at"], name: "id_last_name_deleted", unique: true end - create_table "emails", force: :cascade do |t| + create_table "emails", id: :serial, force: :cascade do |t| t.string "imap_message_id", null: false t.integer "user_id" t.string "mediator_type" @@ -194,7 +194,7 @@ t.index ["mediator_id", "mediator_type"], name: "index_emails_on_mediator_id_and_mediator_type" end - create_table "field_groups", force: :cascade do |t| + create_table "field_groups", id: :serial, force: :cascade do |t| t.string "name", limit: 64 t.string "label", limit: 128 t.integer "position" @@ -205,7 +205,7 @@ t.string "klass_name", limit: 32 end - create_table "fields", force: :cascade do |t| + create_table "fields", id: :serial, force: :cascade do |t| t.string "type" t.integer "field_group_id" t.integer "position" @@ -217,17 +217,17 @@ t.text "collection" t.boolean "disabled" t.boolean "required" - t.integer "maxlength", limit: 4 + t.integer "maxlength" t.datetime "created_at" t.datetime "updated_at" t.integer "pair_id" t.text "settings" - t.integer "minlength", limit: 4, default: 0 + t.integer "minlength", default: 0 t.index ["field_group_id"], name: "index_fields_on_field_group_id" t.index ["name"], name: "index_fields_on_name" end - create_table "groups", force: :cascade do |t| + create_table "groups", id: :serial, force: :cascade do |t| t.string "name" t.datetime "created_at" t.datetime "updated_at" @@ -241,7 +241,27 @@ t.index ["user_id"], name: "index_groups_users_on_user_id" end - create_table "leads", force: :cascade do |t| + create_table "imported_files", id: :serial, force: :cascade do |t| + t.string "filename", limit: 64, default: "", null: false + t.string "md5sum", limit: 32, default: "", null: false + t.datetime "created_at" + t.datetime "updated_at" + end + + create_table "importers", id: :serial, force: :cascade do |t| + t.integer "attachment_file_size" + t.string "attachment_file_name", null: false + t.string "attachment_content_type" + t.string "entity_type", null: false + t.string "entity_id" + t.string "status", default: "new", null: false + t.text "map" + t.text "messages" + t.datetime "created_at" + t.datetime "updated_at" + end + + create_table "leads", id: :serial, force: :cascade do |t| t.integer "user_id" t.integer "campaign_id" t.integer "assigned_to" @@ -273,7 +293,7 @@ t.index ["user_id", "last_name", "deleted_at"], name: "index_leads_on_user_id_and_last_name_and_deleted_at", unique: true end - create_table "lists", force: :cascade do |t| + create_table "lists", id: :serial, force: :cascade do |t| t.string "name" t.text "url" t.datetime "created_at" @@ -282,7 +302,7 @@ t.index ["user_id"], name: "index_lists_on_user_id" end - create_table "opportunities", force: :cascade do |t| + create_table "opportunities", id: :serial, force: :cascade do |t| t.integer "user_id" t.integer "campaign_id" t.integer "assigned_to" @@ -303,7 +323,7 @@ t.index ["user_id", "name", "deleted_at"], name: "id_name_deleted", unique: true end - create_table "permissions", force: :cascade do |t| + create_table "permissions", id: :serial, force: :cascade do |t| t.integer "user_id" t.string "asset_type" t.integer "asset_id" @@ -315,7 +335,7 @@ t.index ["user_id"], name: "index_permissions_on_user_id" end - create_table "preferences", force: :cascade do |t| + create_table "preferences", id: :serial, force: :cascade do |t| t.integer "user_id" t.string "name", limit: 32, default: "", null: false t.text "value" @@ -324,7 +344,7 @@ t.index ["user_id", "name"], name: "index_preferences_on_user_id_and_name" end - create_table "sessions", force: :cascade do |t| + create_table "sessions", id: :serial, force: :cascade do |t| t.string "session_id", null: false t.text "data" t.datetime "created_at" @@ -333,7 +353,7 @@ t.index ["updated_at"], name: "index_sessions_on_updated_at" end - create_table "settings", force: :cascade do |t| + create_table "settings", id: :serial, force: :cascade do |t| t.string "name", limit: 32, default: "", null: false t.text "value" t.datetime "created_at" @@ -341,7 +361,7 @@ t.index ["name"], name: "index_settings_on_name" end - create_table "taggings", force: :cascade do |t| + create_table "taggings", id: :serial, force: :cascade do |t| t.integer "tag_id" t.integer "taggable_id" t.integer "tagger_id" @@ -353,13 +373,13 @@ t.index ["taggable_id", "taggable_type", "context"], name: "index_taggings_on_taggable_id_and_taggable_type_and_context" end - create_table "tags", force: :cascade do |t| + create_table "tags", id: :serial, force: :cascade do |t| t.string "name" t.integer "taggings_count", default: 0 t.index ["name"], name: "index_tags_on_name", unique: true end - create_table "tasks", force: :cascade do |t| + create_table "tasks", id: :serial, force: :cascade do |t| t.integer "user_id" t.integer "assigned_to" t.integer "completed_by" @@ -380,7 +400,7 @@ t.index ["user_id", "name", "deleted_at"], name: "index_tasks_on_user_id_and_name_and_deleted_at", unique: true end - create_table "users", force: :cascade do |t| + create_table "users", id: :serial, force: :cascade do |t| t.string "username", limit: 32, default: "", null: false t.string "email", limit: 254, default: "", null: false t.string "first_name", limit: 32 @@ -423,7 +443,7 @@ t.index ["username", "deleted_at"], name: "index_users_on_username_and_deleted_at", unique: true end - create_table "versions", force: :cascade do |t| + create_table "versions", id: :serial, force: :cascade do |t| t.string "item_type", null: false t.integer "item_id", null: false t.string "event", limit: 512, null: false diff --git a/lib/fat_free_crm.rb b/lib/fat_free_crm.rb index 741b011274..0e74ebe027 100644 --- a/lib/fat_free_crm.rb +++ b/lib/fat_free_crm.rb @@ -52,6 +52,7 @@ def application? require "fat_free_crm/tabs" require "fat_free_crm/callback" require "fat_free_crm/view_factory" +require "fat_free_crm/import_handle" require "activemodel-serializers-xml" require "country_select" diff --git a/lib/fat_free_crm/import_handle.rb b/lib/fat_free_crm/import_handle.rb new file mode 100644 index 0000000000..26b31170b9 --- /dev/null +++ b/lib/fat_free_crm/import_handle.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +# Copyright (c) 2008-2013 Michael Dvorkin and contributors. +# +# Fat Free CRM is freely distributable under the terms of MIT license. +# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php +#------------------------------------------------------------------------------ +require 'roo' +require 'json' + +module FatFreeCRM + class ImportHandle + class << self + def get_columns(path) + headers = {} + xlsx = Roo::Spreadsheet.open(path) + sheet = xlsx.sheet(0) + sheet.row(1).each_with_index do |header, i| + headers[header] = i + end + headers + end + + def get_values(map, sheet, row) + values = {} + map.each do |att, i| + if i.is_a?(Hash) + values[att] = get_values(i, sheet, row) + elsif !i.empty? && (i.to_i >= 0) + value = sheet.row(row)[i.to_i] + values[att] = value + end + end + + values + end + + def process(importer) + errors = [] + map = JSON.parse(importer.map) + xlsx = Roo::Spreadsheet.open(importer.attachment.path) + + xlsx.each_with_pagename do |_name, sheet| + ((sheet.first_row + 1)..sheet.last_row).each do |row| + values = get_values(map, sheet, row) + + # TODO: Do this more geneic + business_address_attributes = {} + if importer.entity_type == 'lead' + values[:campaign_id] = importer.entity_id + business_address_attributes = values.delete('business_address_attributes') if values.key?('business_address_attributes') + end + + item = importer.entity_type.capitalize.constantize.create(values) + if item.valid? + item.save + if importer.entity_type == 'lead' + business_address_attributes["address_type"] = "Business" + business_address_attributes["addressable_type"] = "Lead" + business_address_attributes["addressable_id"] = item.id + address = Address.create(business_address_attributes) + address.save + end + else + errors << item.errors.full_messages + end + end + end + + if errors.empty? + importer.status = :imported + else + importer.status = :error + importer.messages = errors.to_json + end + importer.save + + importer + end + end + end +end diff --git a/spec/factories/imported_files.rb b/spec/factories/imported_files.rb new file mode 100644 index 0000000000..aea1e9c6d3 --- /dev/null +++ b/spec/factories/imported_files.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# Copyright (c) 2008-2013 Michael Dvorkin and contributors. +# +# Fat Free CRM is freely distributable under the terms of MIT license. +# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php +#------------------------------------------------------------------------------ +FactoryBot.define do + factory :imported_file do + filename { "MyString" } + md5sum { "MyString" } + end +end diff --git a/spec/factories/importer.rb b/spec/factories/importer.rb new file mode 100644 index 0000000000..3623feeb8e --- /dev/null +++ b/spec/factories/importer.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Copyright (c) 2008-2013 Michael Dvorkin and contributors. +# +# Fat Free CRM is freely distributable under the terms of MIT license. +# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php +#------------------------------------------------------------------------------ +FactoryBot.define do + factory :importer do + entity_type { :lead } + entity_id { 1 } + attachment_file_size { Random.rand(1..1024) } + attachment_file_name { "#{FFaker::Filesystem.file_name}.#{%w[xls xlsx].sample}" } + attachment_content_type { %w[text/xml application/xml].sample } + status { FFaker::Lorem.word } + created_at { FactoryBot.generate(:time) } + updated_at { FactoryBot.generate(:time) } + end +end diff --git a/spec/models/files/imported_file_spec.rb b/spec/models/files/imported_file_spec.rb new file mode 100644 index 0000000000..986670771d --- /dev/null +++ b/spec/models/files/imported_file_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Copyright (c) 2008-2013 Michael Dvorkin and contributors. +# +# Fat Free CRM is freely distributable under the terms of MIT license. +# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php +#------------------------------------------------------------------------------ +# == Schema Information +# +# Table name: imported_files +# +# id :integer not null, primary key +# filename :string(64) default(""), not null +# md5sum :string(32) default(""), not null +# + +require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper') + +RSpec.describe ImportedFile, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/models/files/importer_spec.rb b/spec/models/files/importer_spec.rb new file mode 100644 index 0000000000..c4c4333431 --- /dev/null +++ b/spec/models/files/importer_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# Copyright (c) 2008-2013 Michael Dvorkin and contributors. +# +# Fat Free CRM is freely distributable under the terms of MIT license. +# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php +#------------------------------------------------------------------------------ +# == Schema Information +# +# Table name: importers +# +# id :integer not null, primary key +# entity_type :string +# entity_id :integer +# attachment_file_size :integer +# attachment_file_name :string(255) +# attachment_content_type :string(255) +# status :string(255) +# created_at :datetime +# updated_at :datetime +# + +require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper') + +RSpec.describe Importer, type: :model do + it "should create a new instance given valid attributes" do + Importer.create!(attributes_for(:importer)) + end + + describe "validates" do + it "attachment" do + is_expected.to have_attached_file(:attachment) + end + + it "attachment presence" do + is_expected.to validate_attachment_presence(:attachment) + end + + xit "attachment file size" do + is_expected.to validate_attachment_size(:attachment) + .less_than(10.megabytes) + end + + it "attachment content type" do + is_expected.to validate_attachment_content_type(:attachment) + .allowing('text/xml', 'application/xml', + 'application/vnd.ms-excel', 'application/x-ole-storage', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') + .rejecting('text/plain') + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index a64548e48a..f68ec05477 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -12,6 +12,7 @@ require 'rspec/rails' require 'capybara/rails' require 'paper_trail/frameworks/rspec' +require "paperclip/matchers" require 'factory_bot_rails' require 'ffaker' @@ -47,6 +48,7 @@ config.include Warden::Test::Helpers config.include DeviseHelpers config.include FeatureHelpers + config.include Paperclip::Shoulda::Matchers Warden.test_mode!