Feature/improve import (#546)

Use async import
BspbOrg · Dec 14, 2023 · a9fa075 · a9fa075
1 parent 0598c70
commit a9fa075
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 94 deletions.
diff --git a/client/scripts/directives/importButtons.js b/client/scripts/directives/importButtons.js
@@ -10,33 +10,23 @@ require('../app').directive('importButtons', /* @ngInject */function ($translate
       const controller = this
       const model = $injector.get($scope.modelId)
 
-      controller.importState = {
-        success: false,
-        importing: true,
-        summary: null
-      }
-
       controller.internalImport = function (items, language, force) {
-        controller.importState.importing = true
-        controller.importState.success = false
-        controller.importState.summary = null
-
         return model.import({ items, skipErrors: force, language }).$promise
           .then(function (res) {
-            controller.importState.success = true
-            controller.importState.summary = res
+            ngToast.create({
+              className: 'success',
+              content: $translate.instant('You will be notified by email when your import is ready')
+            })
           })
           .catch(function (error) {
-            controller.importState.success = false
-            controller.importState.summary = error.data
-          })
-          .finally(function (res) {
-            controller.importState.importing = false
-            $scope.onComplete({ params: 'lalalala' })
+            ngToast.create({
+              className: 'danger',
+              content: '<p>' + $translate.instant('Error during import') + '</p><pre>' + (error && error.data ? error.data.error : JSON.stringify(error, null, 2)) + '</pre>'
+            })
           })
       }
 
-      controller.import = function (inputType, items, language) {
+      controller.import = function (inputType, items, language, ignoreErrors) {
         if (inputType !== 'csv') {
           ngToast.create({
             className: 'danger',
@@ -46,29 +36,7 @@ require('../app').directive('importButtons', /* @ngInject */function ($translate
           return
         }
 
-        $uibModal.open({
-          templateUrl: 'views/import_summary.html',
-          controller: function ($uibModalInstance, state) {
-            this.state = state
-
-            this.close = function () {
-              $uibModalInstance.close()
-            }
-
-            this.forceImport = function () {
-              controller.internalImport(items, language, true)
-            }
-          },
-          controllerAs: 'ctrl',
-          backdrop: 'static',
-          resolve: {
-            state: function () {
-              return controller.importState
-            }
-          }
-        })
-
-        return controller.internalImport(items, language, false)
+        return controller.internalImport(items, language, ignoreErrors)
       }
 
       controller.openImportModal = function () {
@@ -85,7 +53,7 @@ require('../app').directive('importButtons', /* @ngInject */function ($translate
           .result.then(function (result) {
             console.log('result', result)
             if (result && result.records && result.records.length) {
-              controller.import('csv', result.records, result.language)
+              controller.import('csv', result.records, result.language, result.ignoreErrors)
             }
           }, function (reason) {
             // Modal dismissed

diff --git a/client/scripts/directives/importRecordsModal.js b/client/scripts/directives/importRecordsModal.js
@@ -10,7 +10,8 @@ require('../app').directive('importRecordsModal', /* @ngInject */function (ngToa
     },
     link: function (scope, element, attrs, controller) {
       const input = document.getElementById('import-file')
-      input.addEventListener('change', function () {
+
+      const parseFile = function () {
         const file = input.files[0]
 
         scope.$apply(function () {
@@ -22,6 +23,21 @@ require('../app').directive('importRecordsModal', /* @ngInject */function (ngToa
         const reader = new FileReader()
         const records = []
 
+        const startTime = new Date().getTime()
+        let endTime = null
+
+        const constructObject = function (key, parentObj, value) {
+          if (key.split('.').length === 1) {
+            parentObj[key] = value
+            return parentObj
+          }
+
+          const curKey = key.split('.')[0]
+          if (!parentObj[curKey]) { parentObj[curKey] = {} }
+          parentObj[curKey] = constructObject(key.split('.').slice(1).join('.'), parentObj[curKey], value)
+          return parentObj
+        }
+
         reader.onload = function () {
           const csvData = reader.result
           const parser = csvParse.parse(csvData, {
@@ -32,7 +48,11 @@ require('../app').directive('importRecordsModal', /* @ngInject */function (ngToa
           parser.on('readable', function () {
             let record
             while ((record = parser.read()) !== null) {
-              records.push(record)
+              let resultRecord = {}
+              Object.keys(record).forEach(function (key) {
+                resultRecord = constructObject(key, resultRecord, record[key])
+              })
+              records.push(resultRecord)
             }
           })
           parser.on('error', function (err) {
@@ -42,12 +62,16 @@ require('../app').directive('importRecordsModal', /* @ngInject */function (ngToa
             })
           })
           parser.on('end', function () {
+            endTime = new Date().getTime()
+            console.log('end', records.length, endTime - startTime)
             controller.fileReady(records)
           })
         }
 
         reader.readAsText(file)
-      })
+      }
+
+      input.addEventListener('change', parseFile)
     },
     controller: /* @ngInject */function ($scope) {
       const controller = this
@@ -56,6 +80,7 @@ require('../app').directive('importRecordsModal', /* @ngInject */function (ngToa
       controller.records = []
       controller.language = 'bg'
       controller.canImport = false
+      controller.ignoreErrors = false
 
       controller.availableLanguages = Object.keys(languages).map(function (key) {
         return {
@@ -65,22 +90,21 @@ require('../app').directive('importRecordsModal', /* @ngInject */function (ngToa
       })
 
       $scope.$watch('$ctrl.records', function (newValue) {
-        console.log('watch', newValue)
         controller.canImport = newValue && newValue.length > 0
       })
 
       controller.fileReady = function (records) {
         $scope.$apply(function () {
           controller.loading = false
-          console.log('fileReady' + records.length, records)
           controller.records = records
         })
       }
       controller.startImport = function () {
         $scope.close({
           $value: {
             records: controller.records,
-            language: controller.language
+            language: controller.language,
+            ignoreErrors: controller.ignoreErrors
           }
         })
       }

diff --git a/client/views/directives/importrecordsmodal.html b/client/views/directives/importrecordsmodal.html
@@ -26,6 +26,13 @@ <h3 class="modal-title" id="modal-title">
       </div>
     </div>
   </div>
+    <div class="row">
+        <div class="col-md-4">
+            <div class="form-group">
+                <input type="checkbox" id="ignore-errors" ng-model="$ctrl.ignoreErrors">
+                <label for="ignore-errors">{{ 'IMPORT_RECORDS_IGNORE_ERRORS' | translate }}</label>
+            </div>
+        </div>
 </div>
 
 <div ng-if="!ctrl.state.importing" class="modal-footer">

diff --git a/client/views/import_summary.html b/client/views/import_summary.html
diff --git a/i18n/bg.json b/i18n/bg.json
@@ -1333,6 +1333,8 @@
     "IMPORT_RECORDS_CHOOSE_FILE": "Изберете файл",
     "IMPORT_RECORDS_READING_FILE": "Зареждане на файла...",
     "IMPORT_RECORDS_LANGUAGE": "Език",
+    "IMPORT_RECORDS_IGNORE_ERRORS": "Игнорирай грешките",
     "BTN_IMPORT": "Импортирай",
-    "FORM_LIST_BUTTON_CSV_IMPORT_TOOLTIP": "Импорт от CSV файл"
+    "FORM_LIST_BUTTON_CSV_IMPORT_TOOLTIP": "Импорт от CSV файл",
+    "You will be notified by email when your import is ready": "Ще бъдете уведомени с емайл, когато импортът Ви е готов."
 }