diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..0761d81 Binary files /dev/null and b/.DS_Store differ diff --git a/gulpfile.js b/gulpfile.js index f652d1f..8e1abc6 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -31,6 +31,10 @@ gulp.task('index', function() { indexer.perform(); }); +gulp.task('reindex', function () { + indexer.reindex(); +}); + gulp.task('run', ['import', 'process_template', 'index']); gulp.task('default', ['import'], function() { diff --git a/lib/gimporter.js b/lib/gimporter.js index 1a36876..1137707 100644 --- a/lib/gimporter.js +++ b/lib/gimporter.js @@ -12,12 +12,12 @@ var dotenv.config(); var connection = mysql.createConnection({ - host : process.env.DB_HOST || '127.0.0.1', - user : process.env.DB_USER, - password : process.env.DB_PASSWORD, - database : process.env.DB_DATABASE, - port : process.env.DB_PORT || 3306, - dateStrings : true + host: process.env.DB_HOST || '127.0.0.1', + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + database: process.env.DB_DATABASE, + port: process.env.DB_PORT || 3306, + dateStrings: true }); connection.connect(); @@ -48,21 +48,56 @@ function unwrap(childrens) { return escapedChildrens; } +gimporter.updateRecord = function (data,tableName) { + var deferred = Q.defer(); + data.indexado = 'NO'; + data.is_new = 'NO'; + + connection.query('UPDATE ?? SET ? WHERE cms_id = ?', [tableName, data, data.cms_id], function (err, rows, fields) { + if (err) { + throw err; + } + deferred.resolve(); + }); + + return deferred.promise; +}; + +gimporter.addRecord = function (data,tableName) { + var deferred = Q.defer(); + connection.query('INSERT INTO ?? SET ?', [tableName, data], function (err, rows, fields) { + if (err) { + throw err; + } + deferred.resolve(); + }); + return deferred.promise; +}; + gimporter.insertFile = function(data, tableName) { var deferred = Q.defer(); + var self = this; var baseData = { created_at: new Date(), updated_at: new Date() }; - - var data2 = extend(data, baseData); - connection.query('INSERT INTO ' + tableName + ' SET ?', data2, function (err, rows, fields) { + var q = connection.query('SELECT * FROM ?? WHERE cms_id = ? LIMIT 1', [tableName, data.cms_id], function (err, rows) { if (err) { throw err; } - deferred.resolve(); - }); + var data2 = extend(data, baseData); + if(rows[0]){ + self.updateRecord(data2,tableName).then(function () { + deferred.resolve(); + }); + + }else{ + self.addRecord(data2,tableName).then(function () { + deferred.resolve(); + }); + } + }); return deferred.promise; }; @@ -112,8 +147,6 @@ gimporter.getLegislacionData = function($file, file) { data = { titulo_norma: titleEl.text() || '', titulo_norma_html: titleEl.html() || '', - //Add en laravel - //field longtext content_html: contentHtml, content: contentText, number: $view.attr('number'), @@ -164,23 +197,14 @@ gimporter.getJurisprudenciaData = function($file, file) { data = { titulo_norma: titleEl.text() || '', titulo_norma_html: titleEl.html() || '', - //Add en laravel - //field longtext content_html: contentHtml, content: contentText, - //Add Laravel - // size 25 numero_norma: $view.attr('numero'), - //Add Laravel - // size 20 anio: $view.attr('anio') || '', tipo_norma_jurisp: $view.attr('tipo_norma_jurisp'), emisor_jurisp: $view.attr('emisor_jurisp') || '', cms_collection_name: $view.attr('cms_collection_name'), - //Add en Laravel - //cms_collection_id: $view.attr('cms_collection_id'), cms_config_id: $view.attr('cms_config_id'), - //Add en Laravel cms_config 20 cms_config: $view.attr('cms_config'), cms_id: $view.attr('cms_id'), cms_parent_name: $view.attr('cms_parent_name'), diff --git a/lib/indexer.js b/lib/indexer.js index 3042c7a..b0e8bac 100644 --- a/lib/indexer.js +++ b/lib/indexer.js @@ -80,7 +80,6 @@ indexer.indexRows = function (rows, docType) { if (err) { throw err; } - console.log('Finalizó indexación'); deferred.resolve(); }); @@ -102,6 +101,21 @@ function temp(from, total, tableName) { return deferred.promise; } +function temp_reindex(from, total, tableName){ + var deferred = Q.defer(); + indexer.indexRowsLeft( tableName) + .then(function() { + if (from + INDEX_INTERVAL >= total) { + deferred.resolve(); + } else { + var newFrom = from + INDEX_INTERVAL; + temp_reindex(newFrom, total, tableName); + } + }); + return deferred.promise; +} + + indexer.indexTable = function (tableName) { var self = this, docType = DOC_TYPES[tableName]; @@ -120,12 +134,30 @@ indexer.indexTable = function (tableName) { }); }; +indexer.getRowsLeft = function(tableName){ + var self = this, + docType = DOC_TYPES[tableName]; + + connection.query('SELECT count(*) as total FROM ?? WHERE indexado=?', [tableName,'NO'], function(err, results) { + if (err) { + throw err; + } + + var from = 0, + total = results[0].total; + + temp_reindex(from, total, tableName) + .then(function() { + }); + + }); +}; + indexer.indexRowsRange = function(from, tableName) { var self = this, deferred = Q.defer(); - var q = connection.query('SELECT ?? FROM ?? LIMIT ?,?', [INDEX_FIELDS, tableName, from, 100], function (err, rows) { if (err) { throw err; @@ -140,6 +172,40 @@ indexer.indexRowsRange = function(from, tableName) { return deferred.promise; } +indexer.indexRowsLeft = function( tableName) { + var self = this, + deferred = Q.defer(); + + var q = connection.query('SELECT ?? FROM ?? WHERE indexado = ? LIMIT ?,?', [INDEX_FIELDS, tableName, 'NO', 0, 100], function (err, rows) { + if (err) { + throw err; + } + var ids = rows.map(function(a) {return a.id;}); + + self.indexRows(rows, DOC_TYPES[tableName]) + .then(function () { + self.updateRecord(ids,tableName).then(function(){ + deferred.resolve(); + }); + }); + }); + + return deferred.promise; +} + +indexer.updateRecord = function (ids,tableName) { + var deferred = Q.defer(); + var data = {indexado: 'SI'}; + var q = connection.query('UPDATE ?? SET ? WHERE id IN (?)', [tableName, data, ids], function (err, rows, fields) { + if (err) { + throw err; + } + deferred.resolve(); + }); + return deferred.promise; +}; + + indexer.perform = function() { var self = this; @@ -148,4 +214,11 @@ indexer.perform = function() { }); }; +indexer.reindex = function () { + var self = this; + DOCUMENT_TABLES.forEach(function (tableName){ + self.getRowsLeft(tableName); + }); +} + module.exports = indexer; \ No newline at end of file