diff --git a/ChangeLog b/ChangeLog index d9f054ff..7c15b821 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ CHANGES ======= +ADD: search about something relevant in a Service/subservice into Splunk (#196) ADD: Allow subservices names with '-' ADD: anon passwords in debug logs ADD: extra check for getUserDetailsByAdmin to create response diff --git a/DOCKER.md b/DOCKER.md index e805186c..190bfa15 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -46,6 +46,7 @@ As you can see there are several arguments to pass to orchestrator entry point i -cygnushost -mailerhost -ldaphost + -splunkhost -keystoneport -keypassport -orionport @@ -55,12 +56,15 @@ As you can see there are several arguments to pass to orchestrator entry point i -cygnusport -ldapport -mailerport + -splunkport -ldapbasedn -maileruser -mailerpasswd -mailerfrom -mailerto -mongodburi + -splunkuser + -splunkpasswd ``` diff --git a/Dockerfile b/Dockerfile index e86fb44e..1bfc7761 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,6 +50,13 @@ ENV MAILER_TO smtpto ENV MONGODB_URI localhost:27017 +ENV SPLUNK_HOST localhost +ENV SPLUNK_PORT 8089 +ENV SPLUNK_PROTOCOL https +ENV SPLUNK_USER splkuser +ENV SPLUNK_PASSWORD splkpass + + ENV PEP_PASSWORD pep ENV IOTAGENT_PASSWORD iotagent @@ -146,6 +153,14 @@ RUN \ \"to\": \"'$MAILER_TO'\" \ }/g' /opt/orchestrator/settings/dev.py && \ + sed -i ':a;N;$!ba;s/SPLUNK = {[A-Za-z0-9,=@.\-\/\"\n: ]*}/SPLUNK = { \ + \"protocol\": \"'$SPLUNK_PROTOCOL'\", \ + \"host\": \"'$SPLUNK_HOST'\", \ + \"port\": \"'$SPLUNK_PORT'\", \ + \"user\": \"'$SPLUNK_USER'\", \ + \"password\": \"'$SPLUNK_PASSWORD'\" \ +}/g' /opt/orchestrator/settings/dev.py && \ + sed -i ':a;N;$!ba;s/MONGODB = {[A-Za-z0-9,\/\"\n: ]*}/MONGODB = { \ \"URI\": \"mongodb:\/\/'$MONGODB_URI'\" \ }/g' /opt/orchestrator/settings/dev.py && \ @@ -178,6 +193,11 @@ RUN \ sed -i 's/MAILER_FROM=smtpuser/MAILER_FROM='$MAILER_FROM'/g' /opt/orchestrator/bin/orchestrator-entrypoint.sh && \ sed -i 's/MAILER_TO=smtpuser/MAILER_TO='$MAILER_TO'/g' /opt/orchestrator/bin/orchestrator-entrypoint.sh && \ sed -i 's/MONGODB_URI=localhost:27017/MONGODB_URI='$MONGODB_URI'/g' /opt/orchestrator/bin/orchestrator-entrypoint.sh && \ + sed -i 's/SPLUNK_HOST=localhost/SPLUNK_HOST='$SPLUNK_HOST'/g' /opt/orchestrator/bin/orchestrator-entrypoint.sh && \ + sed -i 's/SPLUNK_PORT=587/SPLUNK_PORT='$SPLUNK_PORT'/g' /opt/orchestrator/bin/orchestrator-entrypoint.sh && \ + sed -i 's/SPLUNK_PROTOCOL=https/SPLUNK_PROTOCOL='$SPLUNK_PROTOCOL'/g' /opt/orchestrator/bin/orchestrator-entrypoint.sh && \ + sed -i 's/SPLUNK_USER=smtpuser@yourdomain.com/SPLUNK_USER='$SPLUNK_USER'/g' /opt/orchestrator/bin/orchestrator-entrypoint.sh && \ + sed -i 's/SPLUNK_PASSWORD=yourpassword/SPLUNK_PASSWORD='$SPLUNK_PASSWORD'/g' /opt/orchestrator/bin/orchestrator-entrypoint.sh && \ # Put orchestrator version sed -i 's/ORC_version/'$ORCHESTRATOR_VERSION'/g' /opt/orchestrator/settings/common.py && \ sed -i 's/\${project.version}/'$ORCHESTRATOR_VERSION'/g' /opt/orchestrator/orchestrator/core/banner.txt && \ diff --git a/README.md b/README.md index 3848381a..e6586feb 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Orchestrator is used to: - Activate / deactivate IoT Modules - Retrieve statistics and metrics about API usage - Create, List, Modify LDAP Users +- Search for something relevant in a service (or subservice) Orchestrator is based mainly on: - Python 2.7 needed @@ -41,6 +42,7 @@ Orchestrator relies on these other IoT parts: - OpenLDAP (optional) - Mailer (optional) - MongoDB (optional) +- Splunk (optional) Some of these IoT parts are optional, this means that orchestrator can work without them but excluding the part of feature in which are involved. This way Keystone and Keypass are mandatory to deal with Orchestrator. diff --git a/apiary.apib b/apiary.apib index 12f98c4f..350d283f 100644 --- a/apiary.apib +++ b/apiary.apib @@ -875,9 +875,62 @@ In the cases where "serviceId" is part of URL "SERVICE_NAME" field at body can b } +## Activate IoT Module in a Service [/v1.0/service/{serviceId}/module_activation/{iot_module}] + +### list activated IoT Modules [GET] ++ Parameters + + serviceId (required, `string`) ... Service Id ++ Request (application/json) + { + "SERVICE_NAME":"test_service", + "SERVICE_USER_NAME":"test_admin_service", + "SERVICE_USER_PASSWORD":"passwd", + "SERVICE_USER_TOKEN":"token", + } ++ Response 200 (application/json) + + Body + { + "activated_modules": [ {"name": "STH", "subscriptionid": "23423435", "alias": "HISTORIC"}, + {"name": "PERSEO", "subscriptionid": "34243434", "alias": "RULES"} ] + } + +### activate an IoT Module [POST] ++ Parameters + + serviceId (required, `string`) ... Service Id + + iot_module (optional, `string`) ... IOTModule ++ Request (application/json) + { + "SERVICE_NAME":"test_service", + "SERVICE_USER_NAME":"test_admin_service", + "SERVICE_USER_PASSWORD":"passwd", + "SERVICE_USER_TOKEN":"token", + "IOTMODULE":"STH" + } + Required: [ "iot_module/IOTMODULE" ] ++ Response 201 (application/json) + + Body + { + "subscriptionid": "23423435" + } + +### deactivate an IoT Module [DELETE] ++ Parameters + + serviceId (required, `string`) ... Service id + + iot_module (optional, `string`) ... IOTModule ++ Request (application/json) + { + "SERVICE_NAME":"test_service", + "SERVICE_USER_NAME":"test_admin_service", + "SERVICE_USER_PASSWORD":"passwd", + "SERVICE_USER_TOKEN":"token", + "IOTMODULE":"STH" + } + Required: [ "iot_module/IOTMODULE" ] ++ Response 204 + ## Activate IoT Module in a Sub-service of service [/v1.0/service/{serviceId}/subservice/{subserviceId}/module_activation/{iot_module}] -### list activated IoT Modules [POST] +### list activated IoT Modules [GET] + Parameters + serviceId (required, `string`) ... Service Id + subserviceId (required, `string`) ... SubService Id @@ -1199,3 +1252,50 @@ The following metrics are collected by the component: { "details": "OK" } + +## Search something relevant about a Service in Splunk [/v1.0/service/{serviceId}/relevant/{component}] + +### Search something relevant about a Service [POST] ++ Parameters + + serviceId (required, `string`) ... Service Id + + COMPONENT (required, `string`) ... IoT component (cygnus-ngsi, Orchestrator, Orion, STH, PEPorion, PEPsth, PEPperseo, PEPiotagent, perseo-fe, perseo-core, IoTAgent) + + TRANSACTION_ID (optional, `string`) ... Transaction Id + + CORRELATOR_ID (optional, `string`) ... Correlator Id + + LOG_LEVEL (optional, `string`) ... Log level (INFO, ERROR, WARN, DEBUG, CRITICAL) + + CUSTOM_TEXT (optional, `string`) ... Custom text ++ Request (application/json) + { + "SERVICE_USER_NAME":"test_admin_service", + "SERVICE_USER_PASSWORD":"passwd", + "SERVICE_USER_TOKEN":"token", + "TRANSACTION_ID":"1552008898-556-00000004824", + "CORRELATOR_ID":"630a1245-6f72-4b33-a93e-f151a5e6b098", + "LOG_LEVEL":"ERROR", + "CUSTOM_TEXT":"core dumped" + } ++ Response 200 (application/json) + + Body { } + +## Search something relevant into subservice of a Service in Splunk [/v1.0/service/{serviceId}/subservice/{subserviceId}/relevant/{component}] + +### search something relevant into subservice of a Service [POST] ++ Parameters + + serviceId (required, `string`) ... Service Id + + subserviceId (required, `string`) ... SubService Id + + COMPONENT (required, `string`) ... IoT component (cygnus-ngsi, Orchestrator, Orion, STH, PEPorion, PEPsth, PEPperseo, PEPiotagent, perseo-fe, perseo-core, IoTAgent) + + TRANSACTION_ID (optional, `string`) ... Transaction Id + + CORRELATOR_ID (optional, `string`) ... Correlator Id + + LOG_LEVEL (optional, `string`) ... Log level (INFO, ERROR, WARN, DEBUG, CRITICAL) + + CUSTOM_TEXT (optional, `string`) ... Custom text ++ Request (application/json) + { + "SERVICE_USER_NAME":"test_admin_service", + "SERVICE_USER_PASSWORD":"passwd", + "SERVICE_USER_TOKEN":"token", + "TRANSACTION_ID":"1552008898-556-00000004824", + "CORRELATOR_ID":"630a1245-6f72-4b33-a93e-f151a5e6b098", + "LOG_LEVEL":"ERROR", + "CUSTOM_TEXT":"core dumped" + } ++ Response 200 (application/json) + + Body { } diff --git a/bin/orchestrator-entrypoint.sh b/bin/orchestrator-entrypoint.sh index 287eb1b6..0f9dfbe4 100755 --- a/bin/orchestrator-entrypoint.sh +++ b/bin/orchestrator-entrypoint.sh @@ -58,6 +58,12 @@ MAILER_TO=smtpuser MONGODB_URI='localhost:27017' +SPLUNK_HOST=localhost +SPLUNK_PORT=8089 +SPLUNK_PROTOCOL=https +SPLUNK_USER=user +SPLUNK_PASSWORD=yourpassword + PEP_PASSWORD=pep IOTAGENT_PASSWORD=iotagent @@ -92,6 +98,9 @@ while [[ $# -gt 0 ]]; do -mailerhost) MAILER_HOST=$VALUE ;; + -splunkhost) + SPLUNK_HOST=$VALUE + ;; -keystoneport) KEYSTONE_PORT=$VALUE ;; @@ -119,6 +128,9 @@ while [[ $# -gt 0 ]]; do -mailerport) MAILER_PORT=$VALUE ;; + -splunkport) + SPLUNK_PORT=$VALUE + ;; -ldapbasedn) LDAP_BASEDN=$VALUE ;; @@ -146,6 +158,12 @@ while [[ $# -gt 0 ]]; do -cygnusmultiagent) CYGNUS_MULTIAGENT=$VALUE ;; + -splunkuser) + SPLUNK_USER=$VALUE + ;; + -splunkpasswd) + SPLUNK_PASSWORD=$VALUE + ;; -debuglevel) DEBUG_LEVEL=$VALUE ;; @@ -276,6 +294,14 @@ sed -i ':a;N;$!ba;s/IOTAGENT = {[A-Za-z0-9,=@.\-\/\"\n: ]*}/IOTAGENT = { \ \"password\": \"'$IOTAGENT_PASSWORD'\", \ }/g' /opt/orchestrator/settings/dev.py +sed -i ':a;N;$!ba;s/SPLUNK = {[A-Za-z0-9,=@.\-\/\"\n: ]*}/SPLUNK = { \ + \"protocol\": \"'$SPLUNK_PROTOCOL'\", \ + \"host\": \"'$SPLUNK_HOST'\", \ + \"port\": \"'$SPLUNK_PORT'\", \ + \"user\": \"'$SPLUNK_USER'\", \ + \"password\": \"'$SPLUNK_PASSWORD'\", \ +}/g' /opt/orchestrator/settings/dev.py + if [ "$DEBUG_LEVEL" ]; then diff --git a/src/orchestrator/api/iotconf.py b/src/orchestrator/api/iotconf.py index 11652ccb..c409df84 100644 --- a/src/orchestrator/api/iotconf.py +++ b/src/orchestrator/api/iotconf.py @@ -114,6 +114,21 @@ def __init__(self): "Forcing to use default conf values (127.0.0.1:27017)") self.MONGODB_URI = "mongnodb://127.0.0.1:27017" + try: + self.SPLUNK_PROTOCOL = settings.SPLUNK['protocol'] + self.SPLUNK_HOST = settings.SPLUNK['host'] + self.SPLUNK_PORT = settings.SPLUNK['port'] + self.SPLUNK_USER = settings.SPLUNK['user'] + self.SPLUNK_PASSWORD = settings.SPLUNK['password'] + except KeyError: + logger.warn("SPLUNK endpoint configuration error. " + + "Forcing to use default conf values (localhost)") + self.SPLUNK_HOST = "localhost" + self.SPLUNK_PORT = "8089" + self.SPLUNK_PROTOCOL = "https" + self.SPLUNK_USER = "user" + self.SPLUNK_PASSWORD = "pass" + # Get Django status error from simple HTTP error code def getStatusFromCode(self, code): diff --git a/src/orchestrator/api/relevant_view.py b/src/orchestrator/api/relevant_view.py new file mode 100644 index 00000000..f794afa2 --- /dev/null +++ b/src/orchestrator/api/relevant_view.py @@ -0,0 +1,117 @@ +# +# Copyright 2019 Telefonica Espana +# +# This file is part of IoT orchestrator +# +# IoT orchestrator is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# IoT orchestrator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es +# +# Author: IoT team +# +import time +import jsonschema + +from rest_framework.views import APIView +from rest_framework.response import Response +from rest_framework import status +from rest_framework.exceptions import ParseError + +from django.conf import settings + +from orchestrator.core.flow.Relevant import Relevant +from orchestrator.api import schemas, parsers +from orchestrator.api.iotconf import IoTConf +from orchestrator.api.stats import Stats + +class Relevant_RESTView(APIView, IoTConf): + """ + { Read } Relevant + + """ + schema_name = "Relevant" + parser_classes = (parsers.JSONSchemaParser,) + + def __init__(self): + IoTConf.__init__(self) + + def get(self, request, service_id, component, subservice_id=None): + return self.post(request, service_id, component, subservice_id) + + def post(self, request, service_id, component, subservice_id=None): + service_start = time.time() + response = service_name = subservice_name = flow = None + HTTP_X_AUTH_TOKEN = self.getXAuthToken(request) + CORRELATOR_ID = self.getCorrelatorIdHeader(request) + try: + try: + jsonschema.validate({'COMPONENT': component}, + schemas.json[self.schema_name]) + except (ValueError, jsonschema.exceptions.ValidationError) as error: + raise ParseError(detail=error.message) + request.data # json validation + flow = Relevant(self.KEYSTONE_PROTOCOL, + self.KEYSTONE_HOST, + self.KEYSTONE_PORT, + None, + None, + None, + SPLUNK_PROTOCOL=self.SPLUNK_PROTOCOL, + SPLUNK_HOST=self.SPLUNK_HOST, + SPLUNK_PORT=self.SPLUNK_PORT, + SPLUNK_USER=self.SPLUNK_USER, + SPLUNK_PASSWORD=self.SPLUNK_PASSWORD, + CORRELATOR_ID=CORRELATOR_ID) + CORRELATOR_ID = self.getCorrelatorId(flow, CORRELATOR_ID) + relevant, service_name, subservice_name = flow.getRelevant( + service_id, + None, + subservice_id, + None, + request.data.get("SERVICE_USER_NAME", None), + request.data.get("SERVICE_USER_PASSWORD", None), + request.data.get("SERVICE_USER_TOKEN", HTTP_X_AUTH_TOKEN), + component, + request.data.get("LOG_LEVEL", None), + request.data.get("CORRELATOR_ID", None), + request.data.get("TRANSACTION_ID", None), + request.data.get("CUSTOM_TEXT", None) + ) + result = {} + if 'error' not in relevant: + result = relevant + Stats.num_post_relevant += 1 + response = Response(result, status=status.HTTP_200_OK, + headers={"Fiware-Correlator": CORRELATOR_ID}) + else: + result = relevant + Stats.num_flow_errors += 1 + response = Response(result['error'], + status=self.getStatusFromCode(result['code']), + headers={"Fiware-Correlator": CORRELATOR_ID}) + + except ParseError as error: + Stats.num_api_errors += 1 + response = Response( + 'Input validation error - {0} {1}'.format(error.message, + error.detail), + status=status.HTTP_400_BAD_REQUEST, + headers={"Fiware-Correlator": CORRELATOR_ID} + ) + self.collectMetrics(service_start, service_name, subservice_name, + request, response, flow) + return response + + diff --git a/src/orchestrator/api/schemas.py b/src/orchestrator/api/schemas.py index f28d608d..f3e77e96 100644 --- a/src/orchestrator/api/schemas.py +++ b/src/orchestrator/api/schemas.py @@ -670,10 +670,10 @@ "name": "IOTModuleActivation", "dependencies": { "SERVICE_USER_NAME": [ - "SERVICE_ADMIN_PASSWORD" + "SERVICE_USER_PASSWORD" ], "SERVICE_USER_PASSWORD": [ - "SERVICE_ADMIN_USER", + "SERVICE_USER_NAME", ] }, "properties": { @@ -698,9 +698,9 @@ }, # "required": [ ], }, - ####### + ########### "LdapUser": { - ####### + ########### "name": "LdapUser", "dependencies": { "LDAP_ADMIN_USER": [ @@ -790,5 +790,64 @@ } } # "required": [ ], + }, + ########### + "Relevant": { + ########### + "name": "Relevant", + "dependencies": { + "SERVICE_USER_NAME": [ + "SERVICE_USER_PASSWORD" + ], + "SERVICE_USER_PASSWORD": [ + "SERVICE_USER_NAME", + ] + }, + "properties": { + "SERVICE_USER_NAME": { + "type": "string", + }, + "SERVICE_USER_PASSWORD": { + "type": "string", + }, + "SERVICE_USER_TOKEN": { + "type": "string", + }, + "SERVICE_ID": { + "type": "string", + }, + "SUBSERVICE_ID": { + "type": "string", + }, + "COMPONENT": { + "type": "string", + "enum": ["cygnus-ngsi", + "Orchestrator", + "Orion", + "STH", + "PEPorion", "PEPsth", "PEPperseo", "PEPiotagent", + "perseo-fe", "perseo-core", + "IoTAgent", + "ctxmboard-gasnatural", + "ckan", + "urbo2" + ] + }, + "LOG_LEVEL": { + "type": "string", + "enum": ["INFO", "ERROR", "WARN", "DEBUG", "CRITICAL"] + }, + "CORRELATOR_ID": { + "type": "string", + "pattern": "^([A-Za-z0-9-]+)$", + }, + "TRANSACTION_ID": { + "type": "string", + "pattern": "^([A-Za-z0-9-]+)$", + }, + "CUSTOM_TEXT": { + "type": "string", + } + }, } } diff --git a/src/orchestrator/api/stats.py b/src/orchestrator/api/stats.py index 706c4ccd..615ea5bf 100644 --- a/src/orchestrator/api/stats.py +++ b/src/orchestrator/api/stats.py @@ -84,6 +84,7 @@ class Stats(object): num_get_module_activation = 0 num_post_module_activation = 0 num_delete_module_activation = 0 + num_post_relevant = 0 num_update_loglevel = 0 diff --git a/src/orchestrator/api/urls.py b/src/orchestrator/api/urls.py index 07a24b5d..a4d0dc68 100644 --- a/src/orchestrator/api/urls.py +++ b/src/orchestrator/api/urls.py @@ -45,6 +45,8 @@ from orchestrator.api.ldap_view import (LdapUser_RESTView, LdapAuth_RESTView) +from orchestrator.api.relevant_view import (Relevant_RESTView) + urlpatterns = [ url(r'^service[/]?$', ServiceCreate_RESTView.as_view(), name='new_service_rest_view'), @@ -70,4 +72,6 @@ url(r'^admin/metrics?$', OrchMetrics_RESTView.as_view(), name='orch_metrics_rest_view'), url(r'^ldap/user?$', LdapUser_RESTView.as_view(), name='ldap_user_rest_view'), url(r'^ldap/auth?$', LdapAuth_RESTView.as_view(), name='ldap_auth_rest_view'), + url(r'^service/(?P\w+)/relevant/(?P[-\w]+)$', Relevant_RESTView.as_view(), name='servicerelevant_rest_view'), + url(r'^service/(?P\w+)/subservice/(?P[-\w]+)/relevant/(?P[-\w]+)$', Relevant_RESTView.as_view(), name='subservicerelevant_rest_view'), ] diff --git a/src/orchestrator/common/util.py b/src/orchestrator/common/util.py index c7b3a52f..dcf01903 100644 --- a/src/orchestrator/common/util.py +++ b/src/orchestrator/common/util.py @@ -233,10 +233,10 @@ def rest_request2(self, url, method, user=None, password=None, headers.update({'Fiware-ServicePath': fiware_service_path}) if self.TRANSACTION_ID: - headers.update({'Fiware-Transaction': self.TRANSACTION_ID}) + headers.update({'Fiware-Transaction': str(self.TRANSACTION_ID)}) if self.CORRELATOR_ID: - headers.update({'Fiware-Correlator': self.CORRELATOR_ID}) + headers.update({'Fiware-Correlator': str(self.CORRELATOR_ID)}) res = None diff --git a/src/orchestrator/core/flow/Relevant.py b/src/orchestrator/core/flow/Relevant.py new file mode 100644 index 00000000..22743ded --- /dev/null +++ b/src/orchestrator/core/flow/Relevant.py @@ -0,0 +1,139 @@ +# +# Copyright 2019 Telefonica Espana +# +# This file is part of IoT orchestrator +# +# IoT orchestrator is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# IoT orchestrator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es +# +# Author: IoT team +# +import json + +from orchestrator.core.flow.base import FlowBase +from orchestrator.common.util import ContextFilterService +from orchestrator.common.util import ContextFilterSubService + +class Relevant(FlowBase): + + def getRelevant(self, + SERVICE_ID, + SERVICE_NAME, + SUBSERVICE_ID, + SUBSERVICE_NAME, + USER_NAME, + USER_PASSWORD, + USER_TOKEN, + COMPONENT, + LOG_LEVEL, + CORRELATOR_ID, + TRANSACTION_ID, + CUSTOM_TEXT): + + '''Get something relevant of a domain. + + In case of HTTP error, return HTTP error + + Params: + SERVICE_ID + SUBSERVICE_ID + USER_NAME + USER_PASSWORD + USER_TOKEN + COMPONENT + LOG_LEVEL + CORRELATOR_ID + TRANSACTION_ID + CUSTOM_TEXT + Return: + + ''' + data_log = { + "SERVICE_ID": "%s" % SERVICE_ID, + "SUBSERVICE_ID": "%s" % SUBSERVICE_ID, + "USER_NAME": "%s" % USER_NAME, + "USER_PASSWORD": "%s" % USER_PASSWORD, + "USER_TOKEN": self.get_extended_token(USER_TOKEN), + "COMPONENT": COMPONENT, + "LOG_LEVEL": LOG_LEVEL, + "CORRELATOR_ID": CORRELATOR_ID, + "TRANSACTION_ID": TRANSACTION_ID, + "CUSTOM_TEXT": CUSTOM_TEXT + } + SERVICE_NAME = None + SUBSERVICE_NAME = None + self.logger.debug("FLOW getRelevant invoked with: %s" % json.dumps( + data_log, indent=3) + ) + try: + if not USER_TOKEN: + if not SERVICE_ID: + USER_TOKEN = self.idm.getToken(SERVICE_NAME, + USER_NAME, + USER_PASSWORD) + SERVICE_ID = self.idm.getDomainId(USER_TOKEN, + SERVICE_NAME) + + else: + USER_TOKEN = self.idm.getToken2(SERVICE_ID, + USER_NAME, + USER_PASSWORD) + self.logger.debug("USER_TOKEN=%s" % USER_TOKEN) + + # Ensure SERVICE_NAME and SUBSERVICE_NAME + SERVICE_NAME = self.ensure_service_name(USER_TOKEN, + SERVICE_ID, + SERVICE_NAME) + self.logger.addFilter(ContextFilterService(SERVICE_NAME)) + + if SUBSERVICE_ID and not SUBSERVICE_NAME: + SUBSERVICE_NAME = self.ensure_subservice_name(USER_TOKEN, + SERVICE_ID, + SUBSERVICE_ID, + None) + if SUBSERVICE_NAME: + self.logger.addFilter(ContextFilterSubService(SUBSERVICE_NAME)) + + if self.idm.isTokenAdmin(USER_TOKEN, SERVICE_ID): + self.logger.info("USER_TOKEN is token admin") + OUTPUT = self.splunk.searchRelevant(SERVICE_NAME, + SUBSERVICE_NAME, + COMPONENT, + LOG_LEVEL, + CORRELATOR_ID, + TRANSACTION_ID, + CUSTOM_TEXT) + else: + self.logger.warn("USER_TOKEN is not token admin") + raise Exception("Token is not a admin token") + + except Exception, ex: + error_code = self.composeErrorCode(ex) + self.logError(self.logger, error_code, ex) + return error_code + + # format OUTPUT + #OUTPUT is a { { "preview":false, "offset":137, "result": {} } } ? + + data_log = { + "OUTPUT": OUTPUT + } + self.logger.info("Summary report : %s" % json.dumps(data_log, indent=3)) + + # Consolidate operations metrics into flow metrics + self.collectComponentMetrics() + + return OUTPUT, SERVICE_NAME, SUBSERVICE_NAME diff --git a/src/orchestrator/core/flow/base.py b/src/orchestrator/core/flow/base.py index 9c05754c..a0a8a4f0 100644 --- a/src/orchestrator/core/flow/base.py +++ b/src/orchestrator/core/flow/base.py @@ -31,6 +31,7 @@ from orchestrator.core.openldap import OpenLdapOperations as OpenLdapOperations from orchestrator.core.mailer import MailerOperations as MailerOperations from orchestrator.core.mongo import MongoDBOperations as MongoDBOperations +from orchestrator.core.splunk import SplunkOperations as SplunkOperations from orchestrator.common.util import ContextFilterCorrelatorId from orchestrator.common.util import ContextFilterTransactionId from orchestrator.common.util import ContextFilterService @@ -65,6 +66,11 @@ def __init__(self, MAILER_FROM="smtpuser", MAILER_TO="smtpuser", MONGODB_URI="mongodb://127.0.0.1:27017", + SPLUNK_PROTOCOL="http", + SPLUNK_HOST="localhost", + SPLUNK_PORT="8089", + SPLUNK_USER="splunk_user", + SPLUNK_PASSWORD="splunk_pwd", TRANSACTION_ID=None, CORRELATOR_ID=None): @@ -125,6 +131,14 @@ def __init__(self, CORRELATOR_ID=self.CORRELATOR_ID, TRANSACTION_ID=self.TRANSACTION_ID) + self.splunk = SplunkOperations(SPLUNK_PROTOCOL, + SPLUNK_HOST, + SPLUNK_PORT, + SPLUNK_USER, + SPLUNK_PASSWORD, + CORRELATOR_ID=self.CORRELATOR_ID, + TRANSACTION_ID=self.TRANSACTION_ID) + self.endpoints = {} self.iotmodules_aliases = {} @@ -238,9 +252,11 @@ def ensure_subservice_name(self, USER_TOKEN, SERVICE_ID, SUBSERVICE_ID, def get_extended_token(self, USER_TOKEN): token_extended = USER_TOKEN + self.logger.debug("token_extended %s" % token_extended) if USER_TOKEN: try: token_detail = self.idm.getTokenDetail(USER_TOKEN) + self.logger.debug("token_detail %s" % token_detail) token_extended = { "token": USER_TOKEN, diff --git a/src/orchestrator/core/keystone.py b/src/orchestrator/core/keystone.py index f88652e0..0a2b0364 100644 --- a/src/orchestrator/core/keystone.py +++ b/src/orchestrator/core/keystone.py @@ -1671,6 +1671,6 @@ def isTokenAdmin(self, TOKEN, DOMAIN_ID): # Check Role admin asigned in DOMAIN token_res = self.getTokenDetail(TOKEN) for role in token_res['token']['roles']: - if role['name'] == 'admin': + if role['name'] == 'admin' or 'SubServiceAdmin' in role['name']: return True return False diff --git a/src/orchestrator/core/splunk.py b/src/orchestrator/core/splunk.py new file mode 100644 index 00000000..f87b8e9d --- /dev/null +++ b/src/orchestrator/core/splunk.py @@ -0,0 +1,125 @@ +# +# Copyright 2019 Telefonica Espana +# +# This file is part of IoT orchestrator +# +# IoT orchestrator is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# IoT orchestrator is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es +# +# Author: IoT team +# +import json +import logging + +from orchestrator.common.util import RestOperations + +logger = logging.getLogger('orchestrator_core') + + +class SplunkOperations(object): + ''' + IoT platform: Splunk + ''' + + def __init__(self, + SPLUNK_PROTOCOL=None, + SPLUNK_HOST=None, + SPLUNK_PORT=None, + SPLUNK_USER=None, + SPLUNK_PASSWORD=None, + CORRELATOR_ID=None, + TRANSACTION_ID=None): + + self.SPLUNK_PROTOCOL = SPLUNK_PROTOCOL + self.SPLUNK_HOST = SPLUNK_HOST + self.SPLUNK_PORT = SPLUNK_PORT + self.SPLUNK_USER = SPLUNK_USER + self.SPLUNK_PASSWORD = SPLUNK_PASSWORD + + self.SplunkRestOperations = RestOperations("SPLUNK", + SPLUNK_PROTOCOL, + SPLUNK_HOST, + SPLUNK_PORT, + CORRELATOR_ID, + TRANSACTION_ID) + + def checkSplunk(self): + res = self.SplunkRestOperations.rest_request2( + url='/services/search/jobs', + method='GET', + user=self.SPLUNK_USER, + password=self.SPLUNK_PASSWORD, + data=None) + assert res.status_code == 200, (res.status_code, res.reason) + pass + + def searchRelevant(self, + SERVICE_NAME, + SUBSERVICE_NAME, + COMPONENT, + LOG_LEVEL, + CORRELATOR_ID, + TRANSACTION_ID, + CUSTOM_TEXT): + + search_data = 'search ' + + if (SERVICE_NAME): + search_data += ' srv=%s' % SERVICE_NAME + if (SUBSERVICE_NAME): + search_data += ' subsrv=/%s' % SUBSERVICE_NAME + if (LOG_LEVEL): + search_data += ' lvl=%s' % LOG_LEVEL + if (COMPONENT): + search_data += ' comp=%s' % COMPONENT + if (CORRELATOR_ID): + search_data += ' corr=%s' % CORRELATOR_ID + if (TRANSACTION_ID): + search_data += ' trans=%s' % TRANSACTION_ID + if (CUSTOM_TEXT): + search_data += ' %s' % CUSTOM_TEXT + + search_data = "search=" + search_data + + logger.info("searchRelevant with: %s " % search_data) + + # TODO: add exec_mode="oneshot" + # "earliest_time=" -d "latest_time=" + + res = self.SplunkRestOperations.rest_request2( + url='/servicesNS/admin/search/search/jobs/export?output_mode=json', + method='POST', + user=self.SPLUNK_USER, + password=self.SPLUNK_PASSWORD, + json_data=False, + data=search_data) + + assert res.status_code == 200, (res.status_code, res.reason) + res_entry = res.content.split('\n') + # Check res_entry lengh: only include 10 items + json_body_response = {} + entries = 0 + for entry in res_entry: + entries += 1 + if (len(entry) > 0): + entry_json = json.loads(entry) + json_body_response.update(entry_json) + if (entries == 10): + break + + logger.debug("json response: %s" % json.dumps(json_body_response, + indent=3)) + return json_body_response diff --git a/src/orchestrator/core/startup.py b/src/orchestrator/core/startup.py index cc7509bc..d6b2f4ae 100644 --- a/src/orchestrator/core/startup.py +++ b/src/orchestrator/core/startup.py @@ -34,6 +34,7 @@ from orchestrator.core.perseo import PerseoOperations as PerseoOperations from orchestrator.core.openldap import OpenLdapOperations as OpenLdapOperations from orchestrator.core.mailer import MailerOperations as MailerOperations +from orchestrator.core.splunk import SplunkOperations as SplunkOperations from orchestrator.common.util import ContextFilterCorrelatorId from orchestrator.common.util import ContextFilterTransactionId @@ -150,6 +151,20 @@ def check_endpoints(): except Exception, ex: logger.warn("MAILER endpoint not found: %s" % ex) + # Splunk: optional + SPLUNK_PROTOCOL = settings.SPLUNK['protocol'] + SPLUNK_HOST = settings.SPLUNK['host'] + SPLUNK_PORT = settings.SPLUNK['port'] + SPLUNK_USER = settings.SPLUNK['user'] + SPLUNK_PASSWORD = settings.SPLUNK['password'] + splunk = SplunkOperations(SPLUNK_PROTOCOL, SPLUNK_HOST, SPLUNK_PORT, + SPLUNK_USER, SPLUNK_PASSWORD) + try: + splunk.checkSplunk() + logger.info("SPLUNK endpoint OK") + except Exception, ex: + logger.warn("SPLUNK endpoint not found: %s" % ex) + return "OK" def show_conf(): @@ -159,7 +174,7 @@ def show_conf(): 'ORION', 'PEP', 'PEP_PERSEO', 'SCIM_API_VERSION', 'CYGNUS', 'STH', 'PERSEO', - 'LDAP', 'MAILER', 'MONGODB' + 'LDAP', 'MAILER', 'MONGODB', 'SPLUNK' ] for name in custom_settings_entries: conf[name] = getattr(settings, name) diff --git a/src/settings/common.py b/src/settings/common.py index c30703eb..34d4deb3 100644 --- a/src/settings/common.py +++ b/src/settings/common.py @@ -232,6 +232,7 @@ LDAP = {} MAILER = {} MONGODB = {} +SPLUNK = {} # List of possible IoTModules: persistence services, etc IOTMODULES = [ "CYGNUS", "STH", "PERSEO"] diff --git a/src/settings/dev.py b/src/settings/dev.py index 3b086455..f6a30511 100644 --- a/src/settings/dev.py +++ b/src/settings/dev.py @@ -92,6 +92,14 @@ "URI": "mongodb://localhost:27017" } +SPLUNK = { + "host": "localhost", + "port": "8089", + "protocol":"https", + "user": "splunkuser", + "password": "splunkpassword", +} + PEP = { "user": "pep", "password": "pep"