diff --git a/src/app/app.js b/src/app/app.js index 0700038ad..f846e8602 100755 --- a/src/app/app.js +++ b/src/app/app.js @@ -90,7 +90,6 @@ function (angular, $, _, appLevelRequire) { delete $httpProvider.defaults.headers.common["X-Requested-With"]; }]); - // TODO: add ajax-solr ? var apps_deps = [ 'elasticjs.service', diff --git a/src/app/controllers/dashLoader.js b/src/app/controllers/dashLoader.js index 34f40b068..ec8f61868 100755 --- a/src/app/controllers/dashLoader.js +++ b/src/app/controllers/dashLoader.js @@ -22,12 +22,14 @@ function (angular, _) { }; $scope.showDropdown = function(type) { - var _l = $scope.loader; + // var _l = $scope.loader; + var _l = dashboard.current.loader || $scope.loader; + if(type === 'load') { return (_l.load_elasticsearch || _l.load_gist || _l.load_local); } if(type === 'save') { - return (_l.save_elasticsearch || _l.save_gist || _l.local_local || _l.save_default); + return (_l.save_elasticsearch || _l.save_gist || _l.save_local || _l.save_default); } if(type === 'share') { return (_l.save_temp); @@ -58,13 +60,8 @@ function (angular, _) { ($scope.loader.save_temp_ttl_enable ? ttl : false) ).then( function(result) { - if (DEBUG) { - console.log('result = ',result); - } + if (DEBUG) { console.debug('dashLoader: result = ',result); } - // if(!_.isUndefined(result._id)) { - // alertSrv.set('Dashboard Saved','This dashboard has been saved to Solr as "' + - // result._id + '"','success',5000); // Solr if(!_.isUndefined(result.response.docs[0].id)) { alertSrv.set('Dashboard Saved','This dashboard has been saved to Solr as "' + @@ -102,13 +99,8 @@ function (angular, _) { $scope.elasticsearch_dblist = function(query) { dashboard.elasticsearch_list(query,$scope.loader.load_elasticsearch_size).then( function(result) { - // DEBUG - console.debug("dashLoader : result=",result); + if (DEBUG) { console.debug("dashLoader: result=",result); } - // if(!_.isUndefined(result.hits)) { - // $scope.hits = result.hits.total; - // $scope.elasticsearch.dashboards = result.hits.hits; - // } // Solr if (!_.isUndefined(result.response.docs)) { $scope.hits = result.response.numFound; diff --git a/src/app/dashboards/bettermap.json b/src/app/dashboards/bettermap.json new file mode 100644 index 000000000..ae95831ff --- /dev/null +++ b/src/app/dashboards/bettermap.json @@ -0,0 +1,179 @@ +{ + "title": "Better Map Demo", + "services": { + "query": { + "idQueue": [ + 1 + ], + "list": { + "0": { + "query": "*:*", + "alias": "", + "color": "#7EB26D", + "id": 0, + "pin": false, + "type": "lucene" + } + }, + "ids": [ + 0 + ] + }, + "filter": { + "idQueue": [ + 1 + ], + "list": { + "0": { + "from": "2014-01-31T22:00:00.000Z", + "to": "2014-04-23T09:53:36.013Z", + "field": "start_time", + "type": "time", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + } + }, + "ids": [ + 0 + ] + } + }, + "rows": [ + { + "title": "Options", + "height": "50px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "error": false, + "span": 4, + "editable": true, + "type": "filtering", + "loadingEditor": false + }, + { + "title": "Set time span", + "error": "", + "span": 4, + "editable": true, + "group": [ + "default" + ], + "type": "timepicker", + "mode": "since", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "timespan": "30d", + "timefield": "start_time", + "timeformat": "", + "refresh": { + "enable": false, + "interval": 600, + "min": 3 + }, + "filter_id": 0, + "status": "Stable", + "time": { + "from": "02/01/2014 00:00:00", + "to": "04/23/2014 11:53:36" + } + }, + { + "error": false, + "span": 4, + "editable": true, + "type": "query", + "loadingEditor": false, + "query": "*:*", + "pinned": true, + "history": [ + "*:*" + ], + "remember": 10, + "title": "Search", + "def_type": "" + } + ] + }, + { + "title": "BetterMapDemo", + "height": "650px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "title": "Better Map Demo", + "span": 12, + "editable": true, + "tooltip": "start_station_id", + "group": [ + "default" + ], + "field" : "start_station", + "type" : "bettermap", + "size" : 7000, + "time_field": "start_time", + "lat_start" : 40, + "lat_end" : 45, + "lon_start" : -80, + "lon_end" : -73, + "dist" : 100, + "sort": [ + "start_time", + "desc" + ], + "queries": { + "mode": "all", + "ids": [ + 0 + ], + "custom": "" + }, + "spyable": true, + "error": false + } + ] + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "none", + "pattern": "[logstash-]YYYY.MM.DD", + "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED" + }, + "style": "dark", + "panel_hints": true, + "loader": { + "save_gist": false, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "solr": { + "server": "http://localhost:8983/solr/", + "core_name": "bike-trip-geo" + } +} \ No newline at end of file diff --git a/src/app/dashboards/default.json b/src/app/dashboards/default.json old mode 100644 new mode 100755 index 76a2aa21f..2197c5854 --- a/src/app/dashboards/default.json +++ b/src/app/dashboards/default.json @@ -200,7 +200,7 @@ "ids": [ 0 ], - "query": "q=*&df=message&df=host&df=path&df=type&wt=json&rows=0&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&facet=true&facet.range=event_timestamp&facet.range.start=2014-04-02T05:07:22.000Z/DAY&facet.range.end=2014-04-02T05:22:22.000Z%2B1DAY/DAY&facet.range.gap=%2B10SECOND", + "query": "q=*&wt=json&rows=0&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&facet=true&facet.range=event_timestamp&facet.range.start=2014-04-02T05:07:22.000Z/DAY&facet.range.end=2014-04-02T05:22:22.000Z%2B1DAY/DAY&facet.range.gap=%2B10SECOND", "custom": "" }, "max_rows": 100000, @@ -258,7 +258,7 @@ "ids": [ 0 ], - "query": "q=*&df=message&df=host&df=path&df=type&wt=json&rows=0&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&facet=true&facet.field=message&facet.range=event_timestamp&facet.range.start=2014-04-02T05:07:22.000Z&facet.range.end=2014-04-02T05:22:22.000Z&facet.range.gap=%2B1DAY&facet.limit=100" + "query": "q=*&wt=json&rows=0&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&facet=true&facet.field=message&facet.range=event_timestamp&facet.range.start=2014-04-02T05:07:22.000Z&facet.range.end=2014-04-02T05:22:22.000Z&facet.range.gap=%2B1DAY&facet.limit=100" }, "field": "message", "exclude": [], @@ -320,7 +320,7 @@ "ids": [ 0 ], - "query": "q=*&df=message&df=host&df=path&df=type&wt=json&rows=500&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&sort=id desc" + "query": "q=*&wt=json&rows=500&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&sort=id desc" }, "field_list": true, "status": "Stable", @@ -357,6 +357,8 @@ }, "solr": { "server": "http://localhost:8983/solr/", - "core_name": "collection1" + "core_name": "collection1", + "core_list": [], + "global_params": "&df=message" } } \ No newline at end of file diff --git a/src/app/dashboards/guided.json b/src/app/dashboards/guided.json old mode 100644 new mode 100755 index 76a2aa21f..2197c5854 --- a/src/app/dashboards/guided.json +++ b/src/app/dashboards/guided.json @@ -200,7 +200,7 @@ "ids": [ 0 ], - "query": "q=*&df=message&df=host&df=path&df=type&wt=json&rows=0&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&facet=true&facet.range=event_timestamp&facet.range.start=2014-04-02T05:07:22.000Z/DAY&facet.range.end=2014-04-02T05:22:22.000Z%2B1DAY/DAY&facet.range.gap=%2B10SECOND", + "query": "q=*&wt=json&rows=0&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&facet=true&facet.range=event_timestamp&facet.range.start=2014-04-02T05:07:22.000Z/DAY&facet.range.end=2014-04-02T05:22:22.000Z%2B1DAY/DAY&facet.range.gap=%2B10SECOND", "custom": "" }, "max_rows": 100000, @@ -258,7 +258,7 @@ "ids": [ 0 ], - "query": "q=*&df=message&df=host&df=path&df=type&wt=json&rows=0&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&facet=true&facet.field=message&facet.range=event_timestamp&facet.range.start=2014-04-02T05:07:22.000Z&facet.range.end=2014-04-02T05:22:22.000Z&facet.range.gap=%2B1DAY&facet.limit=100" + "query": "q=*&wt=json&rows=0&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&facet=true&facet.field=message&facet.range=event_timestamp&facet.range.start=2014-04-02T05:07:22.000Z&facet.range.end=2014-04-02T05:22:22.000Z&facet.range.gap=%2B1DAY&facet.limit=100" }, "field": "message", "exclude": [], @@ -320,7 +320,7 @@ "ids": [ 0 ], - "query": "q=*&df=message&df=host&df=path&df=type&wt=json&rows=500&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&sort=id desc" + "query": "q=*&wt=json&rows=500&fq=event_timestamp:[2014-04-02T05:07:22.000Z%20TO%202014-04-02T05:22:22.000Z]&sort=id desc" }, "field_list": true, "status": "Stable", @@ -357,6 +357,8 @@ }, "solr": { "server": "http://localhost:8983/solr/", - "core_name": "collection1" + "core_name": "collection1", + "core_list": [], + "global_params": "&df=message" } } \ No newline at end of file diff --git a/src/app/dashboards/syslog.json b/src/app/dashboards/syslog.json old mode 100644 new mode 100755 index 8f190a44d..ae2d80744 --- a/src/app/dashboards/syslog.json +++ b/src/app/dashboards/syslog.json @@ -175,7 +175,7 @@ "ids": [ 0 ], - "query": "q=error&df=message&df=host&df=path&df=type&wt=json&rows=0&fq=event_timestamp:[2014-04-01T07:47:30.228Z%20TO%202014-04-02T07:47:30.228Z]&facet=true&facet.range=event_timestamp&facet.range.start=2014-04-01T07:47:30.228Z/DAY&facet.range.end=2014-04-02T07:47:30.228Z%2B1DAY/DAY&facet.range.gap=%2B10MINUTE", + "query": "q=error&wt=json&rows=0&fq=event_timestamp:[2014-04-01T07:47:30.228Z%20TO%202014-04-02T07:47:30.228Z]&facet=true&facet.range=event_timestamp&facet.range.start=2014-04-01T07:47:30.228Z/DAY&facet.range.end=2014-04-02T07:47:30.228Z%2B1DAY/DAY&facet.range.gap=%2B10MINUTE", "custom": "" }, "title": "System Log Messages", @@ -213,7 +213,7 @@ "ids": [ 0 ], - "query": "q=error&df=message&df=host&df=path&df=type&wt=json&rows=0&fq=event_timestamp:[2014-04-01T07:47:30.228Z%20TO%202014-04-02T07:47:30.228Z]&facet=true&facet.field=syslog_program&facet.range=event_timestamp&facet.range.start=2014-04-01T07:47:30.228Z&facet.range.end=2014-04-02T07:47:30.228Z&facet.range.gap=%2B1DAY&facet.limit=10", + "query": "q=error&wt=json&rows=0&fq=event_timestamp:[2014-04-01T07:47:30.228Z%20TO%202014-04-02T07:47:30.228Z]&facet=true&facet.field=syslog_program&facet.range=event_timestamp&facet.range.start=2014-04-01T07:47:30.228Z&facet.range.end=2014-04-02T07:47:30.228Z&facet.range.gap=%2B1DAY&facet.limit=10", "custom": "" }, "field": "syslog_program", @@ -279,7 +279,7 @@ "ids": [ 0 ], - "query": "q=error&df=message&df=host&df=path&df=type&wt=json&rows=1000&fq=event_timestamp:[2014-04-01T07:47:30.228Z%20TO%202014-04-02T07:47:30.228Z]&sort=event_timestamp desc", + "query": "q=error&wt=json&rows=1000&fq=event_timestamp:[2014-04-01T07:47:30.228Z%20TO%202014-04-02T07:47:30.228Z]&sort=event_timestamp desc", "custom": "" }, "field_list": true, @@ -301,6 +301,7 @@ "style": "light", "panel_hints": true, "loader": { + "dropdown_collections": false, "save_gist": false, "save_elasticsearch": true, "save_local": true, @@ -316,6 +317,8 @@ }, "solr": { "server": "http://localhost:8983/solr/", - "core_name": "logstash_logs" + "core_name": "logstash_logs", + "core_list": [], + "global_params": "&df=message" } } \ No newline at end of file diff --git a/src/app/directives/kibanaPanel.js b/src/app/directives/kibanaPanel.js index 377a0edd9..15c145e4a 100755 --- a/src/app/directives/kibanaPanel.js +++ b/src/app/directives/kibanaPanel.js @@ -41,10 +41,31 @@ function (angular) { '' + '' + - ''+ '' + + '' + + '' + + '' + + '' + + '' + + '' + + +// '' + +// ''+ +// '' + + '' + ''+ '' + @@ -100,4 +121,4 @@ function (angular) { }; }); -}); \ No newline at end of file +}); diff --git a/src/app/panels/bettermap/editor.html b/src/app/panels/bettermap/editor.html index bcbd304d3..7aebfeb45 100755 --- a/src/app/panels/bettermap/editor.html +++ b/src/app/panels/bettermap/editor.html @@ -2,16 +2,46 @@
Coordinate Field geoJSON array! Long,Lat NOT Lat,Long
- +
Tooltip Field
- +
Max Points
- +
+ +
+
+
+
Starting Longitude
+ +
+
+
+
+
Starting Latitude
+ +
+
+
+ +
+
+
+
Ending Longitude
+ +
+
+
+
+
Ending Latitude
+ +
+
+
\ No newline at end of file diff --git a/src/app/panels/bettermap/module.js b/src/app/panels/bettermap/module.js index 90ad5a26f..620e8a169 100755 --- a/src/app/panels/bettermap/module.js +++ b/src/app/panels/bettermap/module.js @@ -14,7 +14,8 @@ define([ 'underscore', './leaflet/leaflet-src', 'require', - + // './leaflet/plugins', // moving it here causing error in the app, fallback to the old Kibana way. + 'css!./module.css', 'css!./leaflet/leaflet.css', 'css!./leaflet/plugins.css' @@ -22,17 +23,13 @@ define([ function (angular, app, _, L, localRequire) { 'use strict'; + var DEBUG = false; // DEBUG mode + var module = angular.module('kibana.panels.bettermap', []); app.useModule(module); module.controller('bettermap', function($scope, querySrv, dashboard, filterSrv) { $scope.panelMeta = { - editorTabs : [ - { - title: 'Queries', - src: 'app/partials/querySelect.html' - } - ], modals : [ { description: "Inspect", @@ -41,6 +38,12 @@ function (angular, app, _, L, localRequire) { show: $scope.panel.spyable } ], + editorTabs : [ + { + title: 'Queries', + src: 'app/partials/querySelect.html' + } + ], status : "Experimental", description : "Displays geo points in clustered groups on a map. The cavaet for this panel is"+ " that, for better or worse, it does NOT use the terms facet and it does query "+ @@ -53,26 +56,43 @@ function (angular, app, _, L, localRequire) { var _d = { queries : { mode : 'all', - ids : [] + ids : [], + query : '*:*', + custom : '' }, - size : 1000, - spyable : true, - tooltip : "_id", + size : 1000, + spyable : true, + lat_start: '', + lat_end : '', + lon_start: '', + lon_end: '', +// tooltip : "_id", field : null }; - _.defaults($scope.panel,_d); + _.defaults($scope.panel, _d); $scope.requireContext = localRequire; // inorder to use relative paths in require calls, require needs a context to run. Without // setting this property the paths would be relative to the app not this context/file. $scope.init = function() { - $scope.$on('refresh',function(){ + $scope.$on('refresh',function() { $scope.get_data(); }); $scope.get_data(); }; + + $scope.set_refresh = function (state) { + $scope.refresh = state; + }; + + $scope.close_edit = function() { + if($scope.refresh) { + $scope.get_data(); + } + $scope.refresh = false; + }; $scope.get_data = function(segment,query_id) { $scope.require(['./leaflet/plugins'], function () { @@ -82,53 +102,78 @@ function (angular, app, _, L, localRequire) { if(dashboard.indices.length === 0) { return; } - + + // check if [lon,lat] field is defined + // Used for Now Multi-valued field, Looking forward to support (solr spatial search) if(_.isUndefined($scope.panel.field)) { $scope.panel.error = "Please select a field that contains geo point in [lon,lat] format"; return; } - // Determine the field to sort on - var timeField = _.uniq(_.pluck(filterSrv.getByType('time'),'field')); - if(timeField.length > 1) { - $scope.panel.error = "Time field must be consistent amongst time filters"; - } else if(timeField.length === 0) { - timeField = null; - } else { - timeField = timeField[0]; - } + // Solr.js + $scope.sjs.client.server(dashboard.current.solr.server + dashboard.current.solr.core_name); var _segment = _.isUndefined(segment) ? 0 : segment; + // var request = $scope.sjs.Request().indices(dashboard.indices); + $scope.panel.queries.ids = querySrv.idsByMode($scope.panel.queries); // This could probably be changed to a BoolFilter - var boolQuery = $scope.ejs.BoolQuery(); + var boolQuery = $scope.sjs.BoolQuery(); _.each($scope.panel.queries.ids,function(id) { boolQuery = boolQuery.should(querySrv.getEjsObj(id)); }); - var request = $scope.ejs.Request().indices(dashboard.indices[_segment]) - .query($scope.ejs.FilteredQuery( - boolQuery, - filterSrv.getBoolFilter(filterSrv.ids).must($scope.ejs.ExistsFilter($scope.panel.field)) - )) - .fields([$scope.panel.field,$scope.panel.tooltip]) - .size($scope.panel.size); + var request = $scope.sjs.Request().indices(dashboard.indices[_segment]); - if(!_.isNull(timeField)) { - request = request.sort(timeField,'desc'); - } + request = request.query( + $scope.sjs.FilteredQuery( + boolQuery, + filterSrv.getBoolFilter(filterSrv.ids) + )) + .size($scope.panel.size); // Set the size of query result $scope.populate_modal(request); - var results = request.doSearch(); + if (DEBUG) { + console.debug('bettermap:\n\trequest=',request,'\n\trequest.toString()=',request.toString()); + } + // Build Solr query + var fq = '&' + filterSrv.getSolrFq(); + var query_size = $scope.panel.size; + var wt_json = '&wt=json'; + var rows_limit; + var sorting = '&sort=' + filterSrv.getTimeField() + ' desc'; // Only get the latest data, sorted by time field. + + // set the size of query result + if (query_size !== undefined && query_size !== 0) { + rows_limit = '&rows=' + query_size; + } else { // default + rows_limit = '&rows=25'; + } + + if($scope.panel.lat_start && $scope.panel.lat_end && $scope.panel.lon_start && $scope.panel.lon_end && $scope.panel.field) { + fq += '&fq=' + $scope.panel.field + '_0_coordinate:[' + $scope.panel.lat_start + ' TO ' + $scope.panel.lat_end + '] AND ' + $scope.panel.field + '_1_coordinate:[' + $scope.panel.lon_start + ' TO ' + $scope.panel.lon_end + ']'; + } + + // Set the panel's query + $scope.panel.queries.query = querySrv.getQuery(0) + wt_json + rows_limit + fq + sorting; + + // Set the additional custom query + if ($scope.panel.queries.custom != null) { + request = request.setQuery($scope.panel.queries.query + $scope.panel.queries.custom); + } else { + request = request.setQuery($scope.panel.queries.query); + } + + var results = request.doSearch(); // Populate scope when we have results + // Using promises results.then(function(results) { $scope.panelMeta.loading = false; if(_segment === 0) { - $scope.hits = 0; $scope.data = []; query_id = $scope.query_id = new Date().getTime(); } @@ -138,15 +183,14 @@ function (angular, app, _, L, localRequire) { $scope.panel.error = $scope.parse_error(results.error); return; } - + // Check that we're still on the same query, if not stop if($scope.query_id === query_id) { - // Keep only what we need for the set - $scope.data = $scope.data.slice(0,$scope.panel.size).concat(_.map(results.hits.hits, function(hit) { + $scope.data = $scope.data.slice(0,$scope.panel.size).concat(_.map(results.response.docs, function(hit) { return { - coordinates : new L.LatLng(hit.fields[$scope.panel.field][1],hit.fields[$scope.panel.field][0]), - tooltip : hit.fields[$scope.panel.tooltip] + coordinates : new L.LatLng(hit[$scope.panel.field + '_0_coordinate'],hit[$scope.panel.field + '_1_coordinate']), + tooltip : hit[$scope.panel.tooltip] }; })); @@ -155,18 +199,17 @@ function (angular, app, _, L, localRequire) { } $scope.$emit('draw'); - // Get $size results then stop querying + // Searching Solr using Segments if($scope.data.length < $scope.panel.size && _segment+1 < dashboard.indices.length) { - $scope.get_data(_segment+1,$scope.query_id); + $scope.get_data(_segment+1, $scope.query_id); } - }); }); }; $scope.populate_modal = function(request) { - $scope.inspector = angular.toJson(JSON.parse(request.toString()),true); + $scope.inspector = angular.toJson(JSON.parse(request.toString()), true); }; }); @@ -195,19 +238,25 @@ function (angular, app, _, L, localRequire) { function render_panel() { scope.require(['./leaflet/plugins'], function () { scope.panelMeta.loading = false; + L.Icon.Default.imagePath = 'app/panels/bettermap/leaflet/images'; if(_.isUndefined(map)) { map = L.map(attrs.id, { - scrollWheelZoom: false, + scrollWheelZoom: true, center: [40, -86], zoom: 10 }); - L.tileLayer('http://{s}.tile.cloudmade.com/57cbb6ca8cac418dbb1a402586df4528/22677/256/{z}/{x}/{y}.png', { + // Add Change to the tile layer url, because it was returning 403 (forbidden) + // Forbidden because of API Key in cloudmade, so I used osm for now + // osm (open street map) (http://{s}.tile.osm.org/{z}/{x}/{y}.png) + // cloud made (http://{s}.tile.cloudmade.com/57cbb6ca8cac418dbb1a402586df4528/22677/256/{z}/{x}/{y}.png) + L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', { maxZoom: 18, minZoom: 2 }).addTo(map); - layerGroup = new L.MarkerClusterGroup({maxClusterRadius:30}); + + layerGroup = new L.MarkerClusterGroup({maxClusterRadius:50}); } else { layerGroup.clearLayers(); } diff --git a/src/app/panels/histogram/editor.html b/src/app/panels/histogram/editor.html index 109f796c3..609dc15d8 100755 --- a/src/app/panels/histogram/editor.html +++ b/src/app/panels/histogram/editor.html @@ -1,6 +1,7 @@
+
@@ -32,6 +33,7 @@
--> +
Chart Settings
@@ -45,15 +47,23 @@
Chart Settings
-
+
+ +
+
-
+
+
+ + +
+
@@ -78,6 +88,7 @@
Chart Settings
+
Tooltip Settings
diff --git a/src/app/panels/histogram/module.js b/src/app/panels/histogram/module.js index 2a7f65ad1..668682e1f 100755 --- a/src/app/panels/histogram/module.js +++ b/src/app/panels/histogram/module.js @@ -44,7 +44,6 @@ define([ 'jquery.flot.stackpercent' ], function (angular, app, $, _, kbn, moment, timeSeries) { - 'use strict'; var DEBUG = false; // DEBUG mode @@ -77,16 +76,14 @@ function (angular, app, $, _, kbn, moment, timeSeries) { // Set and populate defaults var _d = { mode : 'count', - // time_field : '@timestamp', time_field : 'event_timestamp', queries : { mode : 'all', ids : [], - query : 'q=*:*', + query : '*:*', custom : '' }, - max_rows : 100000, // maximum number of rows returned from Solr - // group_limit : 10000, // maximum number of results to return for each group (group.limit) + max_rows : 100000, // maximum number of rows returned from Solr (also use this for group.limit to simplify UI setting) value_field : null, group_field : null, auto_int : true, @@ -102,6 +99,7 @@ function (angular, app, $, _, kbn, moment, timeSeries) { stack : true, points : false, lines : false, + lines_smooth: false, // Enable 'smooth line' mode by removing zero values from the plot. legend : true, 'x-axis' : true, 'y-axis' : true, @@ -116,16 +114,6 @@ function (angular, app, $, _, kbn, moment, timeSeries) { _.defaults($scope.panel,_d); - // var chart_colors = [ - // "#7EB26D","#EAB839","#6ED0E0","#EF843C","#E24D42","#1F78C1","#BA43A9","#705DA0", //1 - // "#508642","#CCA300","#447EBC","#C15C17","#890F02","#0A437C","#6D1F62","#584477", //2 - // "#B7DBAB","#F4D598","#70DBED","#F9BA8F","#F29191","#82B5D8","#E5A8E2","#AEA2E0", //3 - // "#629E51","#E5AC0E","#64B0C8","#E0752D","#BF1B00","#0A50A1","#962D82","#614D93", //4 - // "#9AC48A","#F2C96D","#65C5DB","#F9934E","#EA6460","#5195CE","#D683CE","#806EB7", //5 - // "#3F6833","#967302","#2F575E","#99440A","#58140C","#052B51","#511749","#3F2B5B", //6 - // "#E0F9D7","#FCEACA","#CFFAFF","#F9E2D2","#FCE2DE","#BADFF4","#F9D9F9","#DEDAF7" //7 - // ]; - $scope.init = function() { // Hide view options by default $scope.options = false; @@ -212,12 +200,11 @@ function (angular, app, $, _, kbn, moment, timeSeries) { // Solr $scope.sjs.client.server(dashboard.current.solr.server + dashboard.current.solr.core_name); - if (DEBUG) { - console.log('histogram:\n\tdashboard=',dashboard,'\n\t$scope=',$scope,'\n\t$scope.panel=',$scope.panel,'\n\tquerySrv=',querySrv,'\n\tfilterSrv=',filterSrv); - } + if (DEBUG) { console.debug('histogram:\n\tdashboard=',dashboard,'\n\t$scope=',$scope,'\n\t$scope.panel=',$scope.panel,'\n\tquerySrv=',querySrv,'\n\tfilterSrv=',filterSrv); } var request = $scope.sjs.Request().indices(dashboard.indices[segment]); $scope.panel.queries.ids = querySrv.idsByMode($scope.panel.queries); + // Build the query _.each($scope.panel.queries.ids, function(id) { var query = $scope.sjs.FilteredQuery( @@ -245,47 +232,23 @@ function (angular, app, $, _, kbn, moment, timeSeries) { $scope.populate_modal(request); // Build Solr query - // TODO: Validate dashboard.current.services.filter.list[0], what if it is not the timestamp field? - // This will cause error. - var start_time = new Date(dashboard.current.services.filter.list[0].from).toISOString(); - var end_time = new Date(dashboard.current.services.filter.list[0].to).toISOString(); - var fq = '&fq=' + $scope.panel.time_field + ':[' + start_time + '%20TO%20' + end_time + ']'; - var df = '&df=message&df=host&df=path&df=type'; + var fq = '&' + filterSrv.getSolrFq(); + var time_field = filterSrv.getTimeField(); + var start_time = filterSrv.getStartTime(); + var end_time = filterSrv.getEndTime(); + var wt_json = '&wt=json'; var rows_limit = '&rows=0'; // for histogram, we do not need the actual response doc, so set rows=0 var facet_gap = $scope.sjs.convertFacetGap($scope.panel.interval); var facet = '&facet=true' + - '&facet.range=' + $scope.panel.time_field + + '&facet.range=' + time_field + '&facet.range.start=' + start_time + '/DAY' + '&facet.range.end=' + end_time + '%2B1DAY/DAY' + '&facet.range.gap=' + facet_gap; - var filter_fq = ''; - var filter_either = []; var values_mode_query = ''; - // Apply filters to the query - _.each(dashboard.current.services.filter.list, function(v,k) { - // Skip the timestamp filter because it's already applied to the query using fq param. - // timestamp filter should be in k = 0 - if (k > 0 && v.field != $scope.panel.time_field && v.active) { - if (DEBUG) { console.log('terms: k=',k,' v=',v); } - if (v.mandate == 'must') { - filter_fq = filter_fq + '&fq=' + v.field + ':"' + v.value + '"'; - } else if (v.mandate == 'mustNot') { - filter_fq = filter_fq + '&fq=-' + v.field + ':"' + v.value + '"'; - } else if (v.mandate == 'either') { - filter_either.push(v.field + ':"' + v.value + '"'); - } - } - }); - // parse filter_either array values, if exists - if (filter_either.length > 0) { - filter_fq = filter_fq + '&fq=(' + filter_either.join(' OR ') + ')'; - } - // For mode = value if($scope.panel.mode === 'values') { - // if (_.isNull($scope.panel.value_field)) { if (!$scope.panel.value_field) { $scope.panel.error = "In " + $scope.panel.mode + " mode a field must be specified"; return; @@ -296,20 +259,16 @@ function (angular, app, $, _, kbn, moment, timeSeries) { facet = ''; // if Group By Field is specified - // if ($scope.panel.group_field && $scope.panel.group_limit) { - // values_mode_query += '&group=true&group.field=' + $scope.panel.group_field + '&group.limit=' + $scope.panel.group_limit; - // } if ($scope.panel.group_field) { values_mode_query += '&group=true&group.field=' + $scope.panel.group_field + '&group.limit=' + $scope.panel.max_rows; } } // Set the panel's query - $scope.panel.queries.query = 'q=' + dashboard.current.services.query.list[0].query + df + wt_json + rows_limit + fq + facet + filter_fq + values_mode_query; + $scope.panel.queries.query = querySrv.getQuery(0) + wt_json + rows_limit + fq + facet + values_mode_query; // Set the additional custom query if ($scope.panel.queries.custom != null) { - // request = request.customQuery($scope.panel.queries.custom); request = request.setQuery($scope.panel.queries.query + $scope.panel.queries.custom); } else { request = request.setQuery($scope.panel.queries.query); @@ -343,10 +302,7 @@ function (angular, app, $, _, kbn, moment, timeSeries) { // Populate scope when we have results results.then(function(results) { - - if (DEBUG) { - console.log('histogram:\n\trequest='+request+'\n\tresults=',results); - } + if (DEBUG) { console.debug('histogram:\n\trequest='+request+'\n\tresults=',results); } $scope.panelMeta.loading = false; if(segment === 0) { @@ -357,7 +313,6 @@ function (angular, app, $, _, kbn, moment, timeSeries) { // Check for error and abort if found if(!(_.isUndefined(results.error))) { - // $scope.panel.error = $scope.parse_error(results.error); $scope.panel.error = $scope.parse_error(results.error.msg); return; } @@ -368,7 +323,7 @@ function (angular, app, $, _, kbn, moment, timeSeries) { var facetIds = [0]; // Need to fix this // Make sure we're still on the same query/queries - // TODO: We probably DON'T NEED THIS + // TODO: We probably DON'T NEED THIS unless we have to support multiple queries in query module. if($scope.query_id === query_id && _.difference(facetIds, $scope.panel.queries.ids).length === 0) { var i = 0, time_series, @@ -377,7 +332,7 @@ function (angular, app, $, _, kbn, moment, timeSeries) { _.each($scope.panel.queries.ids, function(id) { // var query_results = results.facets[id]; - if (DEBUG) { console.log('histogram: i='+i+', results=',results,', segment=',segment,', $scope=',$scope); } + if (DEBUG) { console.debug('histogram: i=',i,', segment=',segment,', $scope=',$scope); } // we need to initialize the data variable on the first run, // and when we are working on the first segment of the data. @@ -389,11 +344,9 @@ function (angular, app, $, _, kbn, moment, timeSeries) { fill_style: 'minimal' }); hits = 0; - if (DEBUG) { console.log('\tfirst run: i='+i+', time_series=',time_series); } + if (DEBUG) { console.debug('\tfirst run: i='+i+', time_series=',time_series); } } else { - if (DEBUG) { - console.log('\tNot first run: i='+i+', $scope.data[i].time_series=',$scope.data[i].time_series,', hits='+$scope.data[i].hits); - } + if (DEBUG) { console.debug('\tNot first run: i='+i+', $scope.data[i].time_series=',$scope.data[i].time_series,', hits='+$scope.data[i].hits); } time_series = $scope.data[i].time_series; // Bug fix for wrong event count: // Solr don't need to accumulate hits count since it can get total count from facet query. @@ -403,13 +356,6 @@ function (angular, app, $, _, kbn, moment, timeSeries) { $scope.hits = 0; } - // push each entry into the time series, while incrementing counters - // _.each(query_results.entries, function(entry) { - // time_series.addValue(entry.time, entry[$scope.panel.mode]); - // hits += entry.count; // The series level hits counter - // $scope.hits += entry.count; // Entire dataset level hits counter - // }); - // Solr facet counts response is in one big array. // So no need to get each segment like Elasticsearch does. if ($scope.panel.mode === 'count') { @@ -446,17 +392,17 @@ function (angular, app, $, _, kbn, moment, timeSeries) { hits += 1; $scope.hits += 1; } - // TESTING + $scope.data[j] = { - // info: querySrv.list[id], - // Need to define chart info here according to the results, cannot use querySrv.list[id] - info: { - alias: groups[j].groupValue, - color: querySrv.colors[j], - - }, - time_series: group_time_series, - hits: hits + // info: querySrv.list[id], + // Need to define chart info here according to the results, cannot use querySrv.list[id] + info: { + alias: groups[j].groupValue, + color: querySrv.colors[j], + + }, + time_series: group_time_series, + hits: hits }; } } else { // Group By Field is not specified @@ -468,22 +414,20 @@ function (angular, app, $, _, kbn, moment, timeSeries) { hits += 1; $scope.hits += 1; } - // TESTING + $scope.data[i] = { - info: querySrv.list[id], - time_series: time_series, - hits: hits + info: querySrv.list[id], + time_series: time_series, + hits: hits }; } } - - if (DEBUG) { console.log('histogram: time_series=',time_series); } if ($scope.panel.mode !== 'values') { $scope.data[i] = { - info: querySrv.list[id], - time_series: time_series, - hits: hits + info: querySrv.list[id], + time_series: time_series, + hits: hits }; } @@ -495,10 +439,9 @@ function (angular, app, $, _, kbn, moment, timeSeries) { // Tell the histogram directive to render. $scope.$emit('render'); - // DON'T NEED THIS + // Don't need this for Solr unless we need to support multiple queries. // If we still have segments left, get them // if(segment < dashboard.indices.length-1) { - // if (DEBUG) { console.log('histogram: Still have segments left!'); } // $scope.get_data(segment+1,query_id); // } } @@ -665,6 +608,24 @@ function (angular, app, $, _, kbn, moment, timeSeries) { scope.data[i].data = scope.data[i].time_series.getFlotPairs(required_times); } + // ISSUE: SOL-76 + // If 'lines_smooth' is enabled, loop through $scope.data[] and remove zero filled entries. + // Without zero values, the line chart will appear smooth as SiLK ;-) + if (scope.panel.lines_smooth) { + for (var i=0; i < scope.data.length; i++) { + var new_data = []; + for (var j=0; j < scope.data[i].data.length; j++) { + // if value of the timestamp !== 0, then add it to new_data + if (scope.data[i].data[j][1] !== 0) { + new_data.push(scope.data[i].data[j]); + } + } + scope.data[i].data = new_data; + } + } + + if (DEBUG) { console.debug('histogram:\n\tflot options = ',options,'\n\tscope.data = ',scope.data); } + scope.plot = $.plot(elem, scope.data, options); } catch(e) { // TODO: Need to fix bug => "Invalid dimensions for plot, width = 0, height = 200" diff --git a/src/app/panels/hits/module.html b/src/app/panels/hits/module.html index 92283dbab..374363dff 100755 --- a/src/app/panels/hits/module.html +++ b/src/app/panels/hits/module.html @@ -33,7 +33,7 @@
-
{{hits}}
+
{{hits}}
@@ -41,4 +41,4 @@

-
\ No newline at end of file +
diff --git a/src/app/panels/hits/module.js b/src/app/panels/hits/module.js index 066a08157..4051795a8 100755 --- a/src/app/panels/hits/module.js +++ b/src/app/panels/hits/module.js @@ -26,6 +26,8 @@ define([ var module = angular.module('kibana.panels.hits', []); app.useModule(module); + var DEBUG = false; // DEBUG mode + module.controller('hits', function($scope, querySrv, dashboard, filterSrv) { $scope.panelMeta = { modals : [ @@ -48,11 +50,14 @@ define([ var _d = { queries : { mode : 'all', - ids : [] + ids : [], + query : '*:*', + basic_query : '', + custom : '' }, style : { "font-size": '10pt'}, arrangement : 'horizontal', - chart : 'bar', + chart : 'total', counter_pos : 'above', donut : false, tilt : false, @@ -80,24 +85,47 @@ define([ return; } + // Solr + $scope.sjs.client.server(dashboard.current.solr.server + dashboard.current.solr.core_name); + var _segment = _.isUndefined(segment) ? 0 : segment; - var request = $scope.ejs.Request().indices(dashboard.indices[_segment]); + + if (DEBUG) { + console.log('hits:\n\tdashboard',dashboard,'\n\tquerySrv=',querySrv,'\n\tfilterSrv=',filterSrv); + } + + var request = $scope.sjs.Request().indices(dashboard.indices); $scope.panel.queries.ids = querySrv.idsByMode($scope.panel.queries); // Build the question part of the query _.each($scope.panel.queries.ids, function(id) { - var _q = $scope.ejs.FilteredQuery( + var _q = $scope.sjs.FilteredQuery( querySrv.getEjsObj(id), filterSrv.getBoolFilter(filterSrv.ids)); request = request - .facet($scope.ejs.QueryFacet(id) + .facet($scope.sjs.QueryFacet(id) .query(_q) ).size(0); }); // Populate the inspector panel - $scope.inspector = angular.toJson(JSON.parse(request.toString()),true); + $scope.populate_modal(request); + + //Solr Search Query + var fq = '&' + filterSrv.getSolrFq(); + var wt_json = '&wt=json'; + var rows_limit = '&rows=0'; // for hits, we do not need the actual response doc, so set rows=0 + var facet = ''; + + $scope.panel.queries.query = querySrv.getQuery(0) + fq + facet + wt_json + rows_limit; + + // Set the additional custom query + if ($scope.panel.queries.custom != null) { + request = request.setQuery($scope.panel.queries.query + $scope.panel.queries.custom); + } else { + request = request.setQuery($scope.panel.queries.query); + } // Then run it var results = request.doSearch(); @@ -105,11 +133,8 @@ define([ // Populate scope when we have results results.then(function(results) { $scope.panelMeta.loading = false; - if(_segment === 0) { - $scope.hits = 0; - $scope.data = []; - query_id = $scope.query_id = new Date().getTime(); - } + $scope.hits = results.response.numFound; + $scope.data = []; // Check for error and abort if found if(!(_.isUndefined(results.error))) { @@ -118,19 +143,20 @@ define([ } // Convert facet ids to numbers - var facetIds = _.map(_.keys(results.facets),function(k){return parseInt(k, 10);}); +// var facetIds = _.map(_.keys(results.facets),function(k){return parseInt(k, 10);}); // Make sure we're still on the same query/queries - if($scope.query_id === query_id && - _.intersection(facetIds,$scope.panel.queries.ids).length === $scope.panel.queries.ids.length - ) { +// if($scope.query_id === query_id && +// _.intersection(facetIds,$scope.panel.queries.ids).length === $scope.panel.queries.ids.length +// ) { var i = 0; - _.each($scope.panel.queries.ids, function(id) { - var v = results.facets[id]; - var hits = _.isUndefined($scope.data[i]) || _segment === 0 ? - v.count : $scope.data[i].hits+v.count; - $scope.hits += v.count; - + var id = $scope.panel.queries.ids[0]; +// _.each($scope.panel.queries.ids, function(id) { +// var v = results.facets[id]; +// var hits = _.isUndefined($scope.data[i]) || _segment === 0 ? +// v.count : $scope.data[i].hits+v.count; +// $scope.hits += v.count; + var hits = $scope.hits; // Create series $scope.data[i] = { info: querySrv.list[id], @@ -139,14 +165,15 @@ define([ data: [[i,hits]] }; - i++; - }); +// i++; +// }); + $scope.$emit('render'); - if(_segment < dashboard.indices.length-1) { - $scope.get_data(_segment+1,query_id); - } +// if(_segment < dashboard.indices.length-1) { +// $scope.get_data(_segment+1,query_id); +// } - } +// } }); }; @@ -161,6 +188,11 @@ define([ $scope.refresh = false; $scope.$emit('render'); }; + + $scope.populate_modal = function(request) { + $scope.inspector = angular.toJson(JSON.parse(request.toString()), true); + }; + }); @@ -185,7 +217,7 @@ define([ elem.css({height:scope.panel.height||scope.row.height}); try { - _.each(scope.data,function(series) { + _.each(scope.data, function(series) { series.label = series.info.alias; series.color = series.info.color; }); @@ -267,4 +299,4 @@ define([ } }; }); -}); \ No newline at end of file +}); diff --git a/src/app/panels/map/editor.html b/src/app/panels/map/editor.html index 95f2912e3..a3bfc481f 100755 --- a/src/app/panels/map/editor.html +++ b/src/app/panels/map/editor.html @@ -1,15 +1,15 @@ -
-
-
-
Field 2 letter country or state code
- -
-
-
-
Max Maximum countries to plot
- -
-
Map
- -
+
+
+
+
Field 2 letter country or state code
+ +
+
+
Max Maximum countries to plot
+ +
+
Map
+ +
+
diff --git a/src/app/panels/map/module.js b/src/app/panels/map/module.js index 8d1d23739..24129e7c9 100755 --- a/src/app/panels/map/module.js +++ b/src/app/panels/map/module.js @@ -8,7 +8,7 @@ element array, jquerymap will generate shades between these colors * size :: How big to make the facet. Higher = more countries * exclude :: Exlude the array of counties - * spyable :: Show the 'eye' icon that reveals the last ES query + * spyable :: Show the 'eye' icon that reveals the last Solr query * index_limit :: This does nothing yet. Eventually will limit the query to the first N indices @@ -19,12 +19,13 @@ define([ 'app', 'underscore', 'jquery', - 'config', './lib/jquery.jvectormap.min' ], function (angular, app, _, $) { 'use strict'; + var DEBUG = false; // DEBUG mode + var module = angular.module('kibana.panels.map', []); app.useModule(module); @@ -44,14 +45,16 @@ function (angular, app, _, $) { status : "Stable", description : "Displays a map of shaded regions using a field containing a 2 letter country "+ ", or US state, code. Regions with more hit are shaded darker. Node that this does use the"+ - " Elasticsearch terms facet, so it is important that you set it to the correct field." + " Solr terms facet, so it is important that you set it to the correct field." }; // Set and populate defaults var _d = { queries : { mode : 'all', - ids : [] + ids : [], + query : '*:*', + custom : '' }, map : "world", colors : ['#A0E2E2', '#265656'], @@ -67,6 +70,18 @@ function (angular, app, _, $) { $scope.get_data(); }; + $scope.set_refresh = function (state) { + $scope.refresh = state; + }; + + $scope.close_edit = function() { + if ($scope.refresh) { + $scope.get_data(); + } + $scope.refresh = false; + $scope.$emit('render'); + }; + $scope.get_data = function() { // Make sure we have everything for the request to complete @@ -77,7 +92,7 @@ function (angular, app, _, $) { var request; - request = $scope.ejs.Request().indices(dashboard.indices); + request = $scope.sjs.Request().indices(dashboard.indices); $scope.panel.queries.ids = querySrv.idsByMode($scope.panel.queries); // This could probably be changed to a BoolFilter @@ -100,16 +115,63 @@ function (angular, app, _, $) { $scope.populate_modal(request); + // Build Solr query + var fq = '&' + filterSrv.getSolrFq(); + var wt_json = '&wt=json'; + var rows_limit = '&rows=0'; // for map module, we don't display results from row, but we use facets. + var facet = '&facet=true' + + '&facet.field=' + $scope.panel.field + + '&facet.limit=' + $scope.panel.size; + + // Set the panel's query + $scope.panel.queries.query = querySrv.getQuery(0) + wt_json + fq + rows_limit + facet; + + // Set the additional custom query + if ($scope.panel.queries.custom != null) { + request = request.setQuery($scope.panel.queries.query + $scope.panel.queries.custom); + } else { + request = request.setQuery($scope.panel.queries.query); + } + + if (DEBUG) { console.debug('map: $scope.panel=',$scope.panel); } + var results = request.doSearch(); // Populate scope when we have results results.then(function(results) { $scope.panelMeta.loading = false; - $scope.hits = results.hits.total; - $scope.data = {}; - _.each(results.facets.map.terms, function(v) { - $scope.data[v.term.toUpperCase()] = v.count; - }); + $scope.data = {}; // empty the data for new results + + if (results.response.numFound) { + $scope.hits = results.response.numFound; + } else { + // Undefined numFound or zero, clear the map. + $scope.$emit('render'); + return false; + } + + if (DEBUG) { console.debug('map: results=',results); } + + var terms = results.facet_counts.facet_fields[$scope.panel.field]; + + if ($scope.hits > 0) { + for (var i=0; i < terms.length; i += 2) { + // Skip states with zero count to make them greyed out in the map. + if (terms[i+1] > 0) { + // if $scope.data[terms] is undefined, assign the value to it + // otherwise, we will add the value. This case can happen when + // the data contains both uppercase and lowercase state letters with + // duplicate states (e.g. CA and ca). By adding the value, the map will + // show correct counts for states with mixed-case letters. + if (!$scope.data[terms[i].toUpperCase()]) { + $scope.data[terms[i].toUpperCase()] = terms[i+1]; + } else { + $scope.data[terms[i].toUpperCase()] += terms[i+1]; + } + } + }; + } + $scope.$emit('render'); }); }; @@ -120,7 +182,9 @@ function (angular, app, _, $) { }; $scope.build_search = function(field,value) { - filterSrv.set({type:'querystring',mandate:'must',query:field+":"+value}); + // Set querystring to both uppercase and lowercase state values with double-quote around the value + // to prevent query error from state=OR (Oregon) + filterSrv.set({type:'querystring',mandate:'must',query:field+':"'+value.toUpperCase()+'" OR '+field+':"'+value.toLowerCase()+'"'}); dashboard.refresh(); }; diff --git a/src/app/panels/query/editor.html b/src/app/panels/query/editor.html index 0f178830a..bc586bb35 100755 --- a/src/app/panels/query/editor.html +++ b/src/app/panels/query/editor.html @@ -1,8 +1 @@ -
-
-
- - -
-
-
\ No newline at end of file +Nothing to Configure Here. \ No newline at end of file diff --git a/src/app/panels/query/module.js b/src/app/panels/query/module.js index e436af5aa..67c75a87a 100755 --- a/src/app/panels/query/module.js +++ b/src/app/panels/query/module.js @@ -27,9 +27,7 @@ define([ // Set and populate defaults var _d = { - // query : "*", query : "*:*", - def_type : '', pinned : true, history : [], remember: 10 // max: 100, angular strap can't take a variable for items param @@ -42,16 +40,6 @@ define([ }; $scope.refresh = function() { - _.each($scope.querySrv.list, function (v) { - if ($scope.panel.def_type) { - // If defType is specified, strip off old defType params from the query - // before appending the new defType value. - v.query = remove_deftype(v.query) + '&defType=' + $scope.panel.def_type; - } else { - // strip off defType (in case previously specified) - v.query = remove_deftype(v.query) - } - }); update_history(_.pluck($scope.querySrv.list,'query')); $rootScope.$broadcast('refresh'); }; @@ -78,12 +66,6 @@ define([ } }; - var remove_deftype = function(query) { - // strip off all defType params in the query - return query.replace(/(&defType=\w+)/g,''); - }; - $scope.init(); - }); }); \ No newline at end of file diff --git a/src/app/panels/table/module.js b/src/app/panels/table/module.js index 6eded1dd9..475bb22ae 100755 --- a/src/app/panels/table/module.js +++ b/src/app/panels/table/module.js @@ -43,6 +43,32 @@ function (angular, app, _, kbn, moment) { show: $scope.panel.spyable } ], +// dropdowns : [ +// { +// description: "Export", +// icon: "icon-save", +// list: [ +// { +// "text": "
Export to File
" +// }, +// { +// "text" : " csv format", +// "href": "", +// "click": "exportfile('csv')" +// }, +// { +// "text": " json format", +// "href": "", +// "click": "exportfile('json')" +// }, +// { +// "text": " xml format", +// "href": "", +// "click": "exportfile('xml')" +// } +// ] +// } +// ], editorTabs : [ { title:'Paging', @@ -53,6 +79,7 @@ function (angular, app, _, kbn, moment) { src: 'app/partials/querySelect.html' } ], + exportfile: true, status: "Stable", description: "A paginated table of records matching your query or queries. Click on a row to "+ "expand it and review all of the fields associated with that document.

" @@ -64,16 +91,14 @@ function (angular, app, _, kbn, moment) { queries : { mode : 'all', ids : [], - query : 'q=*:*', + query : '*:*', + basic_query : '', custom : '' }, size : 100, // Per page pages : 5, // Pages available offset : 0, - - // sort : ['_score','desc'], sort : ['event_timestamp','desc'], - group : "default", style : {'font-size': '9pt'}, overflow: 'min-height', @@ -86,14 +111,14 @@ function (angular, app, _, kbn, moment) { trimFactor: 300, normTimes : true, spyable : true, - time_field : 'event_timestamp' + time_field : 'event_timestamp', + saveOption : 'json' }; _.defaults($scope.panel,_d); $scope.init = function () { $scope.Math = Math; // Solr - // $scope.sjs = $scope.sjs || sjsResource(config.solr + config.solr_collection); $scope.sjs = $scope.sjs || sjsResource(dashboard.current.solr.server + dashboard.current.solr.core_name); $scope.$on('refresh',function(){$scope.get_data();}); @@ -197,12 +222,8 @@ function (angular, app, _, kbn, moment) { var _segment = _.isUndefined(segment) ? 0 : segment; $scope.segment = _segment; - if(DEBUG) { - var dummy = new Date(dashboard.current.services.filter.list[0].from).toISOString(); - console.log('table: Begin of get_data():\n\t$scope=',$scope,'\n\t$scope.panel=',$scope.panel,'\n\t_segment='+_segment,'\n\tdashboard.indices[_segment]=',dashboard.indices[_segment],'\n\tdashboard=',dashboard,'\n\tdashboard.current.services.filter.list[0].from='+ dummy,'\n\tquerySrv=',querySrv,'\n\tfilterSrv=',filterSrv); - } + if (DEBUG) { console.debug('table: Begin of get_data():\n\t$scope=',$scope,'\n\t$scope.panel=',$scope.panel,'\n\t_segment='+_segment,'\n\tdashboard.indices[_segment]=',dashboard.indices[_segment],'\n\tdashboard=',dashboard,'\n\tquerySrv=',querySrv,'\n\tfilterSrv=',filterSrv); } - // Solr $scope.sjs.client.server(dashboard.current.solr.server + dashboard.current.solr.core_name); var request = $scope.sjs.Request().indices(dashboard.indices[_segment]); @@ -225,57 +246,15 @@ function (angular, app, _, kbn, moment) { .size($scope.panel.size*$scope.panel.pages) // Set the size of query result .sort($scope.panel.sort[0],$scope.panel.sort[1]); - $scope.populate_modal(request); + $scope.panel_request = request; - // Create a facet to store and pass on time_field value to request.doSearch() - // var facet = $scope.sjs.RangeFacet('time_facet'); - // facet = facet.field($scope.panel.time_field); - // request = request.facet(facet); + if (DEBUG) { console.debug('table:\n\trequest=',request,'\n\trequest.toString()=',request.toString()); } - if (DEBUG) { - console.log('table:\n\trequest=',request,'\n\trequest.toString()=',request.toString()); - } - - // TODO: Parse query here and send to request.doSearch() - // declare default Solr params here - // get query - // get from and to time range - // get query.size - // construct the query - // set queryData - // request = request.setQuery(q); - // TODO: Validate dashboard.current.services.filter.list[0], what if it is not the timestamp field? - // This will cause error. - var start_time = new Date(dashboard.current.services.filter.list[0].from).toISOString(); - var end_time = new Date(dashboard.current.services.filter.list[0].to).toISOString(); - var fq = '&fq=' + $scope.panel.time_field + ':[' + start_time + '%20TO%20' + end_time + ']'; + var fq = '&' + filterSrv.getSolrFq(); var query_size = $scope.panel.size * $scope.panel.pages; - var df = '&df=message&df=host&df=path&df=type'; var wt_json = '&wt=json'; var rows_limit; var sorting = ''; - var filter_fq = ''; - var filter_either = []; - - // Apply filters to the query - _.each(dashboard.current.services.filter.list, function(v,k) { - // Skip the timestamp filter because it's already applied to the query using fq param. - // timestamp filter should be in k = 0 - if (k > 0 && v.field != $scope.panel.time_field && v.active) { - if (DEBUG) { console.log('terms: k=',k,' v=',v); } - if (v.mandate == 'must') { - filter_fq = filter_fq + '&fq=' + v.field + ':"' + v.value + '"'; - } else if (v.mandate == 'mustNot') { - filter_fq = filter_fq + '&fq=-' + v.field + ':"' + v.value + '"'; - } else if (v.mandate == 'either') { - filter_either.push(v.field + ':"' + v.value + '"'); - } - } - }); - // parse filter_either array values, if exists - if (filter_either.length > 0) { - filter_fq = filter_fq + '&fq=(' + filter_either.join(' OR ') + ')'; - } if ($scope.panel.sort[0] !== undefined && $scope.panel.sort[1] !== undefined) { sorting = '&sort=' + $scope.panel.sort[0] + ' ' + $scope.panel.sort[1]; @@ -284,18 +263,18 @@ function (angular, app, _, kbn, moment) { // set the size of query result if (query_size !== undefined && query_size !== 0) { rows_limit = '&rows=' + query_size; - // facet_limit = '&facet.limit=' + query_size; } else { // default rows_limit = '&rows=25'; - // facet_limit = '&facet.limit=10'; } // Set the panel's query - $scope.panel.queries.query = 'q=' + dashboard.current.services.query.list[0].query + df + wt_json + rows_limit + fq + sorting + filter_fq; + $scope.panel.queries.basic_query = querySrv.getQuery(0) + fq + sorting + rows_limit; + $scope.panel.queries.query = $scope.panel.queries.basic_query + wt_json; + + if (DEBUG) { console.debug('table: query=',$scope.panel.queries.query); } // Set the additional custom query if ($scope.panel.queries.custom != null) { - // request = request.customQuery($scope.panel.queries.custom); request = request.setQuery($scope.panel.queries.query + $scope.panel.queries.custom); } else { request = request.setQuery($scope.panel.queries.query); @@ -316,61 +295,31 @@ function (angular, app, _, kbn, moment) { $scope.data = []; } - if(DEBUG) { - console.log('table: results=',results); - console.log('\t_segment='+_segment+', $scope.hits='+$scope.hits+', $scope.data=',$scope.data,', query_id='+query_id+'\n\t$scope.panel',$scope.panel); - } + if(DEBUG) { console.debug('table:\n\tresults=',results,'\n\t_segment=',_segment,', $scope.hits=',$scope.hits,', $scope.data=',$scope.data,', query_id=',query_id,'\n\t$scope.panel',$scope.panel); } // Check for error and abort if found if(!(_.isUndefined(results.error))) { - // $scope.panel.error = $scope.parse_error(results.error); $scope.panel.error = $scope.parse_error(results.error.msg); // There's also results.error.code return; } // Check that we're still on the same query, if not stop if($scope.query_id === query_id) { - // $scope.data= $scope.data.concat(_.map(results.hits.hits, function(hit) { - $scope.data = $scope.data.concat(_.map(results.response.docs, function(hit) { + $scope.data = $scope.data.concat(_.map(results.response.docs, function(hit) { var _h = _.clone(hit); - //_h._source = kbn.flatten_json(hit._source); - //_h.highlight = kbn.flatten_json(hit.highlight||{}); - - _h.kibana = { - // _source : kbn.flatten_json(hit._source), - // highlight : kbn.flatten_json(hit.highlight||{}) - - // _source : kbn.flatten_json(hit_object), - _source : kbn.flatten_json(hit), - highlight : kbn.flatten_json(hit.highlight||{}) - }; - return _h; - })); + _h.kibana = { + _source : kbn.flatten_json(hit), + highlight : kbn.flatten_json(hit.highlight||{}) + }; + + return _h; + })); // Solr does not need to accumulate hits count because it can get total count // from a single faceted query. - // $scope.hits += results.hits.total; $scope.hits = results.response.numFound; - if (DEBUG) { - console.log('\t$scope.hits='+$scope.hits+', $scope.data=',$scope.data); - } - - // NO NEED for sorting here. Solr result is already sorted. - // Sort the data - // $scope.data = _.sortBy($scope.data, function(v){ - // if(!_.isUndefined(v.sort)) { - // return v.sort[0]; - // } else { - // return 0; - // } - // }); - - // We DO NOT need to reverse here because Solr's result is already sorted. - // if($scope.panel.sort[1] === 'desc') { - // if (DEBUG) { console.log('\tREVERSE IT!!'); } - // $scope.data.reverse(); - // } + if (DEBUG) { console.debug('table: $scope.hits=',$scope.hits,', $scope.data=',$scope.data); } // Keep only what we need for the set $scope.data = $scope.data.slice(0,$scope.panel.size * $scope.panel.pages); @@ -386,14 +335,41 @@ function (angular, app, _, kbn, moment) { _segment+1 < dashboard.indices.length) { $scope.get_data(_segment+1,$scope.query_id); - if (DEBUG) { - console.log('\tnot sorting in reverse chrono order!'); - } + if (DEBUG) { console.debug('\tnot sorting in reverse chrono order!'); } } }); }; + $scope.exportfile = function(filetype) { + var omitHeader = '&omitHeader=true'; + var exportQuery = $scope.panel.queries.basic_query + '&wt=' + filetype + omitHeader; + var request = $scope.panel_request; + request = request.setQuery(exportQuery); + var response = request.doSearch(); + + response.then(function(response) { + var blob; // the file to be written + // TODO: manipulating solr requests + // pagination (batch downloading) + // example: 1,000,000 rows will explode the memory ! + if(filetype === 'json') { + blob = new Blob([angular.toJson(response,true)], {type: "application/json;charset=utf-8"}); + } else if(filetype === 'csv') { + blob = new Blob([response.toString()], {type: "text/csv;charset=utf-8"}); + } else if(filetype === 'xml'){ + blob = new Blob([response.toString()], {type: "application/xml;charset=utf-8"}); + } else { + // incorrect file type + alert('incorrect file type'); + return false; + } + // from filesaver.js + window.saveAs(blob, "table"+"-"+new Date().getTime()+"."+filetype); + return true; + }); + }; + $scope.populate_modal = function(request) { $scope.inspector = angular.toJson(JSON.parse(request.toString()),true); }; @@ -429,7 +405,6 @@ function (angular, app, _, kbn, moment) { return obj; }; - }); // This also escapes some xml sequences @@ -457,8 +432,6 @@ function (angular, app, _, kbn, moment) { }; }); - - module.filter('tableJson', function() { var json; return function(text,prettyLevel) { @@ -506,4 +479,4 @@ function (angular, app, _, kbn, moment) { }; }); -}); \ No newline at end of file +}); diff --git a/src/app/panels/terms/module.js b/src/app/panels/terms/module.js index 5f2953968..cb5cc7489 100755 --- a/src/app/panels/terms/module.js +++ b/src/app/panels/terms/module.js @@ -48,7 +48,7 @@ function (angular, app, _, $, kbn) { queries : { mode : 'all', ids : [], - query : 'q=*:*', + query : '*:*', custom : '' }, field : 'type', @@ -93,9 +93,7 @@ function (angular, app, _, $, kbn) { //Solr $scope.sjs.client.server(dashboard.current.solr.server + dashboard.current.solr.core_name); - if (DEBUG) { - console.log('terms:\n\tdashboard',dashboard,'\n\tquerySrv=',querySrv,'\n\tfilterSrv=',filterSrv); - } + if (DEBUG) { console.debug('terms:\n\tdashboard',dashboard,'\n\tquerySrv=',querySrv,'\n\tfilterSrv=',filterSrv); } request = $scope.sjs.Request().indices(dashboard.indices); @@ -119,64 +117,30 @@ function (angular, app, _, $, kbn) { filterSrv.getBoolFilter(filterSrv.ids) )))).size(0); - // Use request.size to limit the facet query (is this a good idea?) - request = request.size($scope.panel.size); - // Populate the inspector panel $scope.inspector = angular.toJson(JSON.parse(request.toString()),true); - // Create a facet to store and pass on time_field value to request.doSearch() - var facet = $scope.sjs.RangeFacet('time_facet'); - facet = facet.field($scope.panel.time_field); - request = request.facet(facet); - // Build Solr query - // TODO: Validate dashboard.current.services.filter.list[0], what if it is not the timestamp field? - // This will cause error. - var start_time = new Date(dashboard.current.services.filter.list[0].from).toISOString(); - var end_time = new Date(dashboard.current.services.filter.list[0].to).toISOString(); - var fq = '&fq=' + $scope.panel.time_field + ':[' + start_time + '%20TO%20' + end_time + ']'; - // var query_size = $scope.panel.size * $scope.panel.pages; - var df = '&df=message&df=host&df=path&df=type'; + var fq = '&' + filterSrv.getSolrFq(); + var time_field = filterSrv.getTimeField(); + var start_time = filterSrv.getStartTime(); + var end_time = filterSrv.getEndTime(); var wt_json = '&wt=json'; var rows_limit = '&rows=0' // for terms, we do not need the actual response doc, so set rows=0 var facet_gap = '%2B1DAY'; var facet = '&facet=true' + '&facet.field=' + $scope.panel.field + - '&facet.range=' + $scope.panel.time_field + + '&facet.range=' + time_field + '&facet.range.start=' + start_time + '&facet.range.end=' + end_time + '&facet.range.gap=' + facet_gap + '&facet.limit=' + $scope.panel.size; - var filter_fq = ''; - var filter_either = []; - - // Apply filters to the query - _.each(dashboard.current.services.filter.list, function(v,k) { - // Skip the timestamp filter because it's already applied to the query using fq param. - // timestamp filter should be in k = 0 - if (k > 0 && v.field != $scope.panel.time_field && v.active) { - if (DEBUG) { console.log('terms: k=',k,' v=',v); } - if (v.mandate == 'must') { - filter_fq = filter_fq + '&fq=' + v.field + ':"' + v.value + '"'; - } else if (v.mandate == 'mustNot') { - filter_fq = filter_fq + '&fq=-' + v.field + ':"' + v.value + '"'; - } else if (v.mandate == 'either') { - filter_either.push(v.field + ':"' + v.value + '"'); - } - } - }); - // parse filter_either array values, if exists - if (filter_either.length > 0) { - filter_fq = filter_fq + '&fq=(' + filter_either.join(' OR ') + ')'; - } // Set the panel's query - $scope.panel.queries.query = 'q=' + dashboard.current.services.query.list[0].query + df + wt_json + rows_limit + fq + facet + filter_fq; + $scope.panel.queries.query = querySrv.getQuery(0) + wt_json + rows_limit + fq + facet; // Set the additional custom query if ($scope.panel.queries.custom != null) { - // request = request.customQuery($scope.panel.queries.custom); request = request.setQuery($scope.panel.queries.query + $scope.panel.queries.custom); } else { request = request.setQuery($scope.panel.queries.query); @@ -186,21 +150,14 @@ function (angular, app, _, $, kbn) { // Populate scope when we have results results.then(function(results) { - if (DEBUG) { - console.log('terms: results=',results); - } + if (DEBUG) { console.debug('terms: results=',results); } var k = 0; $scope.panelMeta.loading = false; - // $scope.hits = results.hits.total; $scope.hits = results.response.numFound; $scope.data = []; - // _.each(results.facets.terms.terms, function(v) { - // var slice = { label : v.term, data : [[k,v.count]], actions: true}; - // $scope.data.push(slice); - // k = k + 1; - // }); + _.each(results.facet_counts.facet_fields, function(v) { for (var i = 0; i < v.length; i++) { var term = v[i]; @@ -216,12 +173,10 @@ function (angular, app, _, $, kbn) { $scope.data.push({label:'Missing field', // data:[[k,results.facets.terms.missing]],meta:"missing",color:'#aaa',opacity:0}); - // TODO: Hard coded to 0 for now. Solr faceting does not provide 'missing' value. data:[[k,0]],meta:"missing",color:'#aaa',opacity:0}); $scope.data.push({label:'Other values', // data:[[k+1,results.facets.terms.other]],meta:"other",color:'#444'}); - // TODO: Hard coded to 0 for now. Solr faceting does not provide 'other' value. data:[[k+1,0]],meta:"other",color:'#444'}); diff --git a/src/app/panels/timepicker/module.html b/src/app/panels/timepicker/module.html index 786bcc447..498e959af 100755 --- a/src/app/panels/timepicker/module.html +++ b/src/app/panels/timepicker/module.html @@ -34,7 +34,7 @@

- +
diff --git a/src/app/panels/timepicker/module.js b/src/app/panels/timepicker/module.js index d325811b2..97649d2e4 100755 --- a/src/app/panels/timepicker/module.js +++ b/src/app/panels/timepicker/module.js @@ -22,6 +22,8 @@ define([ function (angular, app, _, moment, kbn) { 'use strict'; + var DEBUG = false; // DEBUG mode + var module = angular.module('kibana.panels.timepicker', []); app.useModule(module); @@ -32,14 +34,12 @@ function (angular, app, _, moment, kbn) { " or if you're using time stamped indices, you need one of these" }; - // Set and populate defaults var _d = { status : "Stable", mode : "relative", time_options : ['5m','15m','1h','6h','12h','24h','2d','7d','30d'], timespan : '15m', - // timefield : '@timestamp', timefield : 'event_timestamp', timeformat : "", refresh : { @@ -77,6 +77,7 @@ function (angular, app, _, moment, kbn) { }; break; } + $scope.time.field = $scope.panel.timefield; // These 3 statements basicly do everything time_apply() does set_timepicker($scope.time.from,$scope.time.to); @@ -90,7 +91,6 @@ function (angular, app, _, moment, kbn) { dashboard.refresh(); - // Start refresh timer if enabled if ($scope.panel.refresh.enable) { $scope.set_interval($scope.panel.refresh.interval); @@ -143,10 +143,13 @@ function (angular, app, _, moment, kbn) { }; var update_panel = function() { - // Update panel's string representation of the time object.Don't update if + // Update panel's string representation of the time object. Don't update if // we're in relative mode since we dont want to store the time object in the // json for relative periods if($scope.panel.mode !== 'relative') { + + if (DEBUG) { console.debug('timepicker: update_panel() $scope.time = ',$scope.time,', $scope.timepicker = ',$scope.timepicker); } + $scope.panel.time = { from : $scope.time.from.format("MM/DD/YYYY HH:mm:ss"), to : $scope.time.to.format("MM/DD/YYYY HH:mm:ss"), @@ -184,15 +187,25 @@ function (angular, app, _, moment, kbn) { $scope.time_apply(); }; - // $scope.time_calc = function(){ var from,to; + // If time picker is defined (usually is) TOFIX: Horrible parsing if(!(_.isUndefined($scope.timepicker))) { + if (DEBUG) { + console.debug('timepicker: time_calc() BEFORE $scope.timepicker.from.date moment = ',moment($scope.timepicker.from.date).format('MM/DD/YYYY')); + console.debug('timepicker: time_calc() BEFORE $scope.timepicker.from.time = ',$scope.timepicker.from.time); + } + + // Fix for SILK-4 and SILK-29 bugs + // by using moment.utc() instead of just moment() from = $scope.panel.mode === 'relative' ? moment(kbn.time_ago($scope.panel.timespan)) : - moment(moment($scope.timepicker.from.date).format('MM/DD/YYYY') + " " + $scope.timepicker.from.time,'MM/DD/YYYY HH:mm:ss'); + moment(moment.utc($scope.timepicker.from.date).format('MM/DD/YYYY') + " " + $scope.timepicker.from.time,'MM/DD/YYYY HH:mm:ss'); to = $scope.panel.mode !== 'absolute' ? moment() : - moment(moment($scope.timepicker.to.date).format('MM/DD/YYYY') + " " + $scope.timepicker.to.time,'MM/DD/YYYY HH:mm:ss'); + moment(moment.utc($scope.timepicker.to.date).format('MM/DD/YYYY') + " " + $scope.timepicker.to.time,'MM/DD/YYYY HH:mm:ss'); + + if (DEBUG) { console.debug('timepicker: time_calc() AFTER calculated from = ',from,',to = ',to); } + // Otherwise (probably initialization) } else { from = $scope.panel.mode === 'relative' ? moment(kbn.time_ago($scope.panel.timespan)) : @@ -205,9 +218,18 @@ function (angular, app, _, moment, kbn) { from = moment(to.valueOf() - 1000); } - $timeout(function(){ - set_timepicker(from,to); - }); + // Fix for SILK-4 and SILK-29 bugs + // This $timeout function causes the timepicker skip-back-one-day bugs. + // Because it will set the timepicker values to $scope.time.from and $scope.time.to, which + // are the calculated UTC time values, and depending on your browser timezone, these values + // might be minus one day. And when you press the timepicker button to update the time, it + // will keep decreasing the date by one day. + // $scope.timepicker => time based on browser timezone + // $scope.time => calculated UTC time + // + // $timeout(function(){ + // set_timepicker(from,to); + // }); return { from : from, @@ -216,23 +238,39 @@ function (angular, app, _, moment, kbn) { }; $scope.time_apply = function() { - $scope.panel.error = ""; // Update internal time object - + $scope.panel.error = ""; + // Remove all other time filters filterSrv.removeByType('time'); + if (DEBUG) { console.debug('timepicker: time_apply() BEFORE time_calc() $scope.time = ',$scope.time,' $scope.timepicker.from.date = ',$scope.timepicker.from.date.toString()); } $scope.time = $scope.time_calc(); $scope.time.field = $scope.panel.timefield; + if (DEBUG) { console.debug('timepicker: time_apply() AFTER time_calc() $scope.time = ',$scope.time,' $scope.timepicker = ',$scope.timepicker); } + update_panel(); set_time_filter($scope.time); dashboard.refresh(); + }; + // No need to automatically call time_apply() when changing time mode, + // because it will mess up the timepicker. + // + // $scope.$watch('panel.mode', $scope.time_apply); + + $scope.time_check = function() { + // FOR DEBUGGING timepicker weirdness with UTC time conversion. + if (DEBUG) { + console.debug('timepicker: time_check() $scope.timepicker = ',$scope.timepicker, + ' $scope.timepicker.from.date = ',$scope.timepicker.from.date.toString(), + ' moment.utc($scope.timepicker.from.date).format("MM/DD/YYYY") = ', moment.utc($scope.timepicker.from.date).format('MM/DD/YYYY') + ); + } }; - $scope.$watch('panel.mode', $scope.time_apply); function set_time_filter(time) { time.type = 'time'; diff --git a/src/app/partials/dashLoader.html b/src/app/partials/dashLoader.html index 0caf24189..428ccc6d1 100755 --- a/src/app/partials/dashLoader.html +++ b/src/app/partials/dashLoader.html @@ -1,18 +1,20 @@ + +
    • - +
    • -
    • +
    • Local File Load dashboard JSON layout from file

      • -
    • +
    • Gist Enter a gist number or url

      @@ -26,7 +28,7 @@
      No gist dashboards found
      • -
    • +
    • Solr
      @@ -44,20 +46,21 @@
      No dashboards matching your query
    • +
    • +
    • - +
    • diff --git a/src/app/partials/dasheditor.html b/src/app/partials/dasheditor.html index 9e7e8be32..e49b3ce83 100755 --- a/src/app/partials/dasheditor.html +++ b/src/app/partials/dasheditor.html @@ -103,46 +103,50 @@
    Default Index If
    +
    +
    Show Collections Picker
    + +
    Allow saving to
    - +
    - +
    - +
    - +
    Allow loading from
    - +
    - +
    - +
    -
    - +
    +
    Sharing
    - +
    -
    - +
    +
    -
    - +
    +
    @@ -153,17 +157,19 @@

    Solr Settings

    -
    Solr Server
    +
    Solr Server
    +
    -
    Collection
    +
    Collection
    +
    - -
    diff --git a/src/app/partials/inspector.html b/src/app/partials/inspector.html index 140273d86..35026fe17 100755 --- a/src/app/partials/inspector.html +++ b/src/app/partials/inspector.html @@ -1,23 +1,19 @@
    - -

    Parameters of Last Query

    +

    Parameters of Last Query

    -
    - + -->
    Panel Query
    {{panel.queries.query}}
    Additional Custom Query
    {{panel.queries.custom}}
    -
    diff --git a/src/app/services/dashboard.js b/src/app/services/dashboard.js index 0a3911e85..762b117ff 100755 --- a/src/app/services/dashboard.js +++ b/src/app/services/dashboard.js @@ -11,8 +11,7 @@ define([ function (angular, $, kbn, _, config, moment, Modernizr) { 'use strict'; - // DEBUG mode - var DEBUG = false; + var DEBUG = false; // DEBUG mode var module = angular.module('kibana.services'); @@ -29,8 +28,9 @@ function (angular, $, kbn, _, config, moment, Modernizr) { rows: [], services: {}, loader: { + dropdown_collections: false, save_gist: false, - save_elasticsearch: true, // TODO: Remove it + save_elasticsearch: true, save_local: true, save_default: true, save_temp: true, @@ -43,16 +43,15 @@ function (angular, $, kbn, _, config, moment, Modernizr) { hide: false }, index: { - interval: 'none', + interval: 'none', // this will always be none because we disable 'Index Settings' tab in dasheditor.html pattern: '_all', // TODO: Remove it default: 'INDEX_MISSING' }, - // collection: { - // name: config.solr_collection - // } solr: { server: config.solr, - core_name: config.solr_core + core_name: config.solr_core, + core_list: [], + global_params: '' } }; @@ -123,12 +122,25 @@ function (angular, $, kbn, _, config, moment, Modernizr) { // Since the dashboard is responsible for index computation, we can compute and assign the indices // here before telling the panels to refresh this.refresh = function() { + // Retrieve Solr collections for the dashboard + kbnIndex.collections(self.current.solr.server).then(function (p) { + if (DEBUG) { console.debug('dashboard: kbnIndex.collections p = ',p); } + if (p.length > 0) { + self.current.solr.core_list = p; + } else { + // No collections returned from Solr + alertSrv.set('No collections','There were no collections returned from Solr.','info',5000); + } + }); + if(self.current.index.interval !== 'none') { if(filterSrv.idsByType('time').length > 0) { var _range = filterSrv.timeRange('min'); kbnIndex.indices(_range.from,_range.to, self.current.index.pattern,self.current.index.interval ).then(function (p) { + if (DEBUG) { console.debug('dashboard: p = ',p); } + if(p.length > 0) { self.indices = p; } else { @@ -143,6 +155,7 @@ function (angular, $, kbn, _, config, moment, Modernizr) { return false; } } + $rootScope.$broadcast('refresh'); }); } else { @@ -159,6 +172,8 @@ function (angular, $, kbn, _, config, moment, Modernizr) { self.indices = [self.current.index.default]; $rootScope.$broadcast('refresh'); } + + if (DEBUG) { console.debug('dashboard: after refresh',self); } }; var dash_defaults = function(dashboard) { @@ -334,41 +349,6 @@ function (angular, $, kbn, _, config, moment, Modernizr) { }); }; - // this.elasticsearch_save = function(type,title,ttl) { - // // Clone object so we can modify it without influencing the existing obejct - // var save = _.clone(self.current); - // var id; - - // // Change title on object clone - // if (type === 'dashboard') { - // id = save.title = _.isUndefined(title) ? self.current.title : title; - // } - - // // Create request with id as title. Rethink this. - // // TODO: - // var request = ejs.Document(config.kibana_index,type,id).source({ - // user: 'guest', - // group: 'guest', - // title: save.title, - // dashboard: angular.toJson(save) - // }); - - // request = type === 'temp' && ttl ? request.ttl(ttl) : request; - - // return request.doIndex( - // // Success - // function(result) { - // if(type === 'dashboard') { - // $location.path('/dashboard/elasticsearch/'+title); - // } - // return result; - // }, - // // Failure - // function() { - // return false; - // } - // ); - // }; // TODO: Rename to solr_save this.elasticsearch_save = function(type,title,ttl) { // Clone object so we can modify it without influencing the existing obejct @@ -380,9 +360,6 @@ function (angular, $, kbn, _, config, moment, Modernizr) { id = save.title = _.isUndefined(title) ? self.current.title : title; } - // DEBUG - // console.log('id for saving dashboard = '+id); - // Create request with id as title. Rethink this. // Use id instead of _id, because it is the default field of Solr schema-less. var request = sjs.Document(config.kibana_index,type,id).source({ @@ -396,8 +373,7 @@ function (angular, $, kbn, _, config, moment, Modernizr) { request = type === 'temp' && ttl ? request.ttl(ttl) : request; - // Solr - // set sjs.client.server to use 'kibana-int' for saving dashboard + // Solr: set sjs.client.server to use 'kibana-int' for saving dashboard sjs.client.server(config.solr + config.kibana_index); return request.doIndex( @@ -432,10 +408,6 @@ function (angular, $, kbn, _, config, moment, Modernizr) { // Solr this.elasticsearch_list = function(query,count) { - // DEBUG - console.log('LINE 411: query = '+query); - console.log('LINE 412: count = '+count); - // set indices and type sjs.client.server(config.solr + config.kibana_index); var request = sjs.Request().indices(config.kibana_index).types('dashboard'); diff --git a/src/app/services/fields.js b/src/app/services/fields.js index 2d7805e75..f64a7fe33 100755 --- a/src/app/services/fields.js +++ b/src/app/services/fields.js @@ -28,10 +28,6 @@ function (angular, _) { } else { self.list = mapFields(_.pick(self.mapping,n)); } - // DEBUG - // console.log("indices: "+indices.toString()); - // console.log("mapping:");console.log(self.mapping); - // console.log("list:"); console.log(self.list); } }); @@ -76,8 +72,6 @@ function (angular, _) { // } this.map = function(indices) { - // delete $http.defaults.headers.common['X-Requested-With']; - var request = $http({ // Query ES to get mapping fields // url: config.elasticsearch + "/" + indices.join(',') + "/_mapping", diff --git a/src/app/services/filterSrv.js b/src/app/services/filterSrv.js index 80ef79184..c87d185e0 100755 --- a/src/app/services/filterSrv.js +++ b/src/app/services/filterSrv.js @@ -5,6 +5,8 @@ define([ ], function (angular, _, config) { 'use strict'; + var DEBUG = false; // DEBUG mode + var module = angular.module('kibana.services'); module.service('filterSrv', function(dashboard, ejsResource, sjsResource) { @@ -21,7 +23,6 @@ define([ // For convenience // var ejs = ejsResource(config.elasticsearch); var solrserver = dashboard.current.solr.server + dashboard.current.solr.core_name || config.solr + config.solr_core; - // var sjs = sjsResource(config.solr + config.solr_core); var sjs = sjsResource(solrserver); var _f = dashboard.current.services.filter; @@ -131,6 +132,112 @@ define([ } }; + // Return fq string for constructing a query to send to Solr. + this.getSolrFq = function() { + var start_time, end_time, time_field; + var filter_fq =''; + var filter_either = []; + + // Loop through the list to find the time field, usually it should be in self.list[0] + _.each(self.list, function(v,k) { + + if (DEBUG) { console.debug('filterSrv: v=',v,' k=',k); } + + if (v.type == 'time') { + time_field = v.field; + start_time = new Date(v.from).toISOString(); + end_time = new Date(v.to).toISOString(); + } else if (v.type == 'terms') { + if (v.mandate == 'must') { + filter_fq = filter_fq + '&fq=' + v.field + ':"' + v.value + '"'; + } else if (v.mandate == 'mustNot') { + filter_fq = filter_fq + '&fq=-' + v.field + ':"' + v.value + '"'; + } else if (v.mandate == 'either') { + filter_either.push(v.field + ':"' + v.value + '"'); + } + } else if (v.type == 'field') { + // v.query contains double-quote around it. + if (v.mandate == 'must') { + filter_fq = filter_fq + '&fq=' + v.field + ':' + v.query; + } else if (v.mandate == 'mustNot') { + filter_fq = filter_fq + '&fq=-' + v.field + ':' + v.query; + } else if (v.mandate == 'either') { + filter_either.push(v.field + ':' + v.query); + } + } else if (v.type == 'querystring') { + if (v.mandate == 'must') { + filter_fq = filter_fq + '&fq=' + v.query; + } else if (v.mandate == 'mustNot') { + filter_fq = filter_fq + '&fq=-' + v.query; + } else if (v.mandate == 'either') { + filter_either.push(v.query); + } + } else { + // Unsupport filter type + return false; + } + }); + // Return false for undefined time field + if (!start_time || !end_time || !time_field) { + return false; + } + // parse filter_either array values, if exists + if (filter_either.length > 0) { + filter_fq = filter_fq + '&fq=(' + filter_either.join(' OR ') + ')'; + } + + return 'fq=' + time_field + ':[' + start_time + '%20TO%20' + end_time + ']' + filter_fq; + }; + + // Get time field for Solr query + this.getTimeField = function() { + var time_field; + _.each(self.list, function(v) { + if (v.type == 'time') { + time_field = v.field; + return; + } + }); + return time_field; + } + + // Get start time for Solr query (e.g. facet.range.start) + this.getStartTime = function() { + var start_time; + _.each(self.list, function(v) { + if (v.type == 'time') { + start_time = new Date(v.from).toISOString(); + return; + } + }); + return start_time; + }; + + // Get end time for Solr query (e.g. facet.range.end) + this.getEndTime = function() { + var end_time; + _.each(self.list, function(v) { + if (v.type == 'time') { + end_time = new Date(v.to).toISOString(); + return; + } + }); + return end_time; + }; + + // Get both start and end time in one shot + this.getStartTimeAndEndTime = function() { + var start_time, end_time; + _.each(self.list, function(v) { + if (v.type == 'time') { + start_time = new Date(v.from).toISOString(); + end_time = new Date(v.to).toISOString(); + return; + } + }); + return [start_time, end_time]; + }; + this.getByType = function(type,inactive) { return _.pick(self.list,self.idsByType(type,inactive)); }; diff --git a/src/app/services/kbnIndex.js b/src/app/services/kbnIndex.js index 56b05ae55..e6d692a5d 100755 --- a/src/app/services/kbnIndex.js +++ b/src/app/services/kbnIndex.js @@ -6,6 +6,7 @@ define([ ], function (angular, _, config, moment) { 'use strict'; + var DEBUG = false; //DEBUG mode var module = angular.module('kibana.services'); @@ -28,6 +29,40 @@ function (angular, _, config, moment) { }); }; + // Solr: returns a promise containing an array of all collections in the Solr server. + // param: solr_server (e.g. http://localhost:8983/solr/) + this.collections = function(solr_server) { + return all_collections(solr_server).then(function (p) { + return p; + }); + } + + // returns a promise containing an array of all collections in Solr + // param: solr_server (e.g. http://localhost:8983/solr/) + function all_collections(solr_server) { + var something = $http({ + // Use Solr Admin handler to get the list of all collections. + // TODO: Need to test this with SolrCloud and LWS + url: solr_server + "admin/cores?action=STATUS&wt=json&omitHeader=true", + method: "GET" + }).error(function(data, status) { + alertSrv.set('Error',"Could not retrieve collections from Solr (error status = "+status+")"); + console.debug('kbnIndex: error data = ',data); + }); + + return something.then(function (p) { + // Parse Solr response to an array of collections + var collections = []; + + _.each(p.data.status, function(v,k) { + collections.push(k); + }); + + if (DEBUG) { console.debug('kbnIndex: all_collections response p = ',p,'collections = ',collections); } + return collections; + }); + } + // returns a promise containing an array of all indices in an elasticsearch // cluster function all_indices() { @@ -64,9 +99,8 @@ function (angular, _, config, moment) { }); return something.then(function(p) { - if (DEBUG) {console.debug('kbnIndex: p=',p);} + if (DEBUG) { console.debug('kbnIndex: p=',p); } - // var indices = []; // _.each(p.data, function(v,k) { // indices.push(k); @@ -75,6 +109,7 @@ function (angular, _, config, moment) { // indices.push(k); // }); // }); + var indices = []; var timestamp_array = p.data.facet_counts.facet_ranges.event_timestamp.counts; @@ -87,8 +122,8 @@ function (angular, _, config, moment) { // indices[] should be in this format // indices = ['logstash-2013.11.25']; - // DEBUG - if (DEBUG) {console.debug('kbnIndex: indices=',indices);} + + if (DEBUG) { console.debug('kbnIndex: indices=',indices); } return indices; }); diff --git a/src/app/services/querySrv.js b/src/app/services/querySrv.js index da18b4839..153ebb8b3 100755 --- a/src/app/services/querySrv.js +++ b/src/app/services/querySrv.js @@ -6,6 +6,8 @@ define([ function (angular, _, config) { 'use strict'; + var DEBUG = false; // DEBUG mode + var module = angular.module('kibana.services'); module.service('querySrv', function(dashboard, ejsResource, sjsResource) { @@ -19,16 +21,15 @@ function (angular, _, config) { // Defaults for query objects var _query = { - // query: '*', query: '*:*', alias: '', pin: false, - type: 'lucene' }; // For convenience - var ejs = ejsResource(config.elasticsearch); - var sjs = sjsResource(config.solr + config.solr_core); + // var ejs = ejsResource(config.elasticsearch); + var solrserver = dashboard.current.solr.server + dashboard.current.solr.core_name || config.solr + config.solr_core; + var sjs = sjsResource(solrserver); var _q = dashboard.current.services.query; this.colors = [ @@ -114,6 +115,24 @@ function (angular, _, config) { } }; + // Get query string for Solr with defType param and default fields (df). + this.getQuery = function(id) { + var solr_q = 'q=' + self.list[id].query; + + // if (self.list[id].type) { + // solr_q += '&defType=' + self.list[id].type; + // } + // if (self.list[id].df) { + // solr_q += '&' + self.list[id].df; + // } + + if (dashboard.current.solr.global_params) { + solr_q += dashboard.current.solr.global_params; + } + + return solr_q; + } + this.findQuery = function(queryString) { return _.findWhere(self.list,{query:queryString}); }; diff --git a/src/config.js b/src/config.js index 0ca02c53c..2e5023e82 100755 --- a/src/config.js +++ b/src/config.js @@ -9,15 +9,16 @@ function (Settings) { return new Settings({ /** - * URL to your elasticsearch server. You almost certainly don't - * want 'http://localhost:9200' here. Even if Kibana and ES are on + * URL to your Solr server. You almost certainly don't + * want 'http://localhost:8983/solr/' here. Even if Banana and Solr are on * the same host * - * By default this will attempt to reach ES at the same host you have - * elasticsearch installed on. You probably want to set it to the FQDN of your - * elasticsearch host + * By default this will attempt to reach Solr at the same host you have + * Banana installed on. You probably want to set it to the FQDN of your + * Solr host * @type {String} */ + // TODO: Remove ES related settings //elasticsearch: "http://"+window.location.hostname+":9200", elasticsearch: "http://localhost:9200", @@ -26,7 +27,7 @@ function (Settings) { solr_core: "logstash_logs", /** - * The default ES index to use for storing Kibana specific object + * The default Solr index to use for storing Banana specific object * such as stored dashboards * @type {String} */ @@ -39,21 +40,22 @@ function (Settings) { */ panel_names: [ 'histogram', - // 'map', // TODO + 'map', // 'pie', // Deprecated, use terms panel instead 'table', - 'filtering', // TODO + 'filtering', 'timepicker', 'text', // 'fields', // Deprecated, table panel now integrates a field selector - // 'hits', // TODO + 'hits', // TODO // 'dashcontrol', // Deprecated, moved to nav bar 'column', // 'derivequeries', // TODO // 'trends', // TODO - // 'bettermap', // TODO + 'bettermap', 'query', - 'terms' + 'terms', + // 'dummy' // Dummy module for testing ] }); }); diff --git a/src/img/silklogo.png b/src/img/silklogo.png new file mode 100644 index 000000000..91c51c159 Binary files /dev/null and b/src/img/silklogo.png differ diff --git a/src/index.html b/src/index.html index 323f3e3ea..603160179 100755 --- a/src/index.html +++ b/src/index.html @@ -33,7 +33,10 @@
    - {{dashboard.current.title}} + + + {{dashboard.current.title}} +
    diff --git a/src/vendor/solrjs/solr-angular-client.js b/src/vendor/solrjs/solr-angular-client.js index 9bbf0be36..3a31686f2 100644 --- a/src/vendor/solrjs/solr-angular-client.js +++ b/src/vendor/solrjs/solr-angular-client.js @@ -19,8 +19,7 @@ angular.module('solrjs.service', []) var DEBUG = false; // DEBUG mode var - - // use existing ejs object if it exists + // use existing sjs object if it exists sjs = window.sjs || {}, /* results are returned as a promise */ @@ -39,7 +38,7 @@ angular.module('solrjs.service', []) url = ''; } - /* implement the elastic.js client interface for angular */ + /* implement the solr.js client interface for angular */ sjs.client = { server: function (s) { if (s == null) { @@ -53,43 +52,17 @@ angular.module('solrjs.service', []) var config = {}; var isUpdate = path.indexOf('/update'); - if(DEBUG) { - console.log('solr-angular-client: url='+url+', path='+path+', isUpdate='+isUpdate); - } + if (DEBUG) { console.debug('solr-angular-client: url=',url,', path=',path,', isUpdate=',isUpdate); } if (isUpdate !== -1) { - // update request, meaning to save a dashboard to kibana-int collection. - // Need to modify path accordingly. - // TODO: Find a better way to implement this. - // path = url.substr(0, url.indexOf('logstash_logs')) + 'kibana-int' + path; config = { headers: {'Content-type':'application/json'} }; } else { - // path = url + path; config = { headers: {'Content-type':'application/x-www-form-urlencoded'} }; } - path = url + path; - - - - // var config = { - // headers: { - // 'Content-type': 'application/x-www-form-urlencoded' - // } - // }; - // For update JSON - // 'Content-type': 'application/json' - - // return promiseThen($http.post(path, data), successcb, errorcb); - - - // SOLR - TESTING Multiple Queries - - // var queries = [data, 'q=*:*&wt=json']; + path = url + path; - if(DEBUG) { - console.log('solr-angular-client: url='+url+', path='+path+', data=',data); - } + if (DEBUG) { console.debug('solr-angular-client: url=',url,', path=',path,', data=',data); } return promiseThen($http.post(path, data, config), successcb, errorcb); }, @@ -115,41 +88,8 @@ angular.module('solrjs.service', []) (errorcb || angular.noop)(undefined); return undefined; }); - }, - // post_solr: function (path, data, successcb, errorcb) { - // var config = {}; - // var isUpdate = path.indexOf('/update'); - - // if (DEBUG) { - // console.log('solr-angular-client: path='+path+', isUpdate='+isUpdate); - // } - - // if (isUpdate !== -1) { - // // update request, meaning to save a dashboard to kibana-int collection. - // // Need to modify path accordingly. - // // TODO: Find a better way to implement this. - // path = url.substr(0, url.indexOf('logstash_logs')) + 'kibana-int' + path; - // config = { headers: {'Content-type':'application/json'} }; - // } else { - // path = url + path; - // config = { headers: {'Content-type':'application/x-www-form-urlencoded'} }; - // } - - // if (DEBUG) { - // console.log('solr-angular-client: path='+path+', data=',data); - // } - - // // var config = { - // // headers: { - // // 'Content-type': 'application/x-www-form-urlencoded' - // // } - // // }; - // // For update JSON - // // 'Content-type': 'application/json' + } - // // return promiseThen($http.post(path, data), successcb, errorcb); - // return promiseThen($http.post(path, data, config), successcb, errorcb); - // } }; return sjs; diff --git a/src/vendor/solrjs/solr.js b/src/vendor/solrjs/solr.js index f73f8c48e..131881d8d 100644 --- a/src/vendor/solrjs/solr.js +++ b/src/vendor/solrjs/solr.js @@ -8881,9 +8881,7 @@ // Otherwise, SolrException "Unknown Command" will occur. params.source = [params.source]; - if (DEBUG) { - console.log('solrjs: params.source = ',params.source); - } + if (DEBUG) { console.debug('solrjs: params.source = ',params.source); } var url = '/update?commit=true', data = JSON.stringify(params.source), @@ -8898,9 +8896,7 @@ url = url + '?' + paramStr; } - if (DEBUG) { - console.log('solrjs: url='+url+', data=',data); - } + if (DEBUG) { console.debug('solrjs: url=',url,', data=',data); } // // do post if id not set so one is created // if (id == null) {