diff --git a/app/assets/mongo.js b/app/assets/mongo.js new file mode 100644 index 0000000..3e71f1e --- /dev/null +++ b/app/assets/mongo.js @@ -0,0 +1,34 @@ +var msg; +// get database stats +var getDbStats = function(db, callback) { +db.command({'dbStats': 1}, +function(err, results) { + console.log(results); + callback(); + return results; +} +); +}; + +var MongoClient = require('mongodb').MongoClient + , assert = require('assert'); + + + +// Connection URL +var url = 'mongodb://spiderrd:spiderrd@45.32.68.44:37017/bilibili_spider'; + + +MongoClient.connect(url, function (err, client) { + assert.equal(null, err); + + var db = client.db('bilibili_spider'); + + msg = getDbStats(db, function() { + client.close(); + }); +}); + + + + diff --git a/app/assets/vue-app.js b/app/assets/vue-app.js new file mode 100644 index 0000000..92a8799 --- /dev/null +++ b/app/assets/vue-app.js @@ -0,0 +1,39 @@ +var MongoClient = require('mongodb').MongoClient + , assert = require('assert'); + + +var vm = new Vue({ +el: '#data-html', +data: { + url : 'mongodb://spiderrd:spiderrd@45.32.68.44:37017/bilibili_spider', + loaded: "loaded", + dataStats: "" + }, + mounted: function () { + this.loaded = "loaded database"; + }, + methods: { + getDbStats: function(db, callback) { + db.command({'dbStats': 1}, + function(err, results) { + console.log(results); + callback(); + return results; + } + ); + } + }, + computed: { + dataStats: MongoClient.connect(url, function (err, client) { + assert.equal(null, err); + + var db = client.db('bilibili_spider'); + + stats = getDbStats(db, function() { + client.close(); + }).objects; + + return stats + }) + } +}) \ No newline at end of file diff --git a/app/bilicrawler.js b/app/bilicrawler.js index d8efc8c..7f0fe64 100644 --- a/app/bilicrawler.js +++ b/app/bilicrawler.js @@ -1,5 +1,10 @@ const loghtml = document.getElementById('log-process') +/////////////////// +// For Kernel +/////////////////// + +//version 20180225-1 const superagent = require('superagent'); var moment = require('moment'); moment.locale('zh-cn'); @@ -74,13 +79,13 @@ const packageFetchInsertAsync = async (pid, mids) => { break // 结束本次任务，尝试下个任务 } } - await sleep(12000) + await sleep(10000) if (cardList.length === midSize) { await uploadPackageAsync(pid, cardList) console.log(`${nowstr()} Send package ${pid}`); } else { - console.error(`${nowstr()} failed to fetch info, mids=${mids}`); - logit(`${nowstr()} failed to fetch info, mids=${mids}`); + console.error(`${nowstr()} failed to fetch info，finished/all=${cardList.length}/${midSize}, mids=${mids}`); + logit(`${nowstr()} failed to fetch info，finished/all=${cardList.length}/${midSize}, mids=${mids}`); } } @@ -105,7 +110,9 @@ const run = async () => { // start code // run(); - +/////////////////// +// For Electron +/////////////////// // Define function that export result to html tag function logit(elem) { diff --git a/app/index.html b/app/index.html index bf10b75..2432d45 100755 --- a/app/index.html +++ b/app/index.html @@ -6,14 +6,6 @@ BiliSpider - @@ -42,12 +34,11 @@

- -
- +
+ +
-
@@ -55,14 +46,27 @@

LOG: 
       
-
-

+ + + - + + +