diff --git a/Dockerfile b/Dockerfile index 459c83b..968cd37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,8 @@ RUN pip3 install --upgrade pip>=10.0.0 &&\ COPY ./src /src -RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@${F8A_WORKER_VERSION} +#RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@${F8A_WORKER_VERSION} +RUN pip3 install git+https://github.com/samuzzal-choudhury/fabric8-analytics-worker.git@289ae6e ADD scripts/entrypoint.sh /bin/entrypoint.sh diff --git a/cico_run_tests.sh b/cico_run_tests.sh index fdf1af9..545730b 100755 --- a/cico_run_tests.sh +++ b/cico_run_tests.sh @@ -8,5 +8,7 @@ set -ex build_image push_image +chmod +x ./runtests.sh +cat ./runtests.sh ./runtests.sh diff --git a/openshift/template.yaml b/openshift/template.yaml index c0b0625..6751ff7 100644 --- a/openshift/template.yaml +++ b/openshift/template.yaml @@ -73,6 +73,21 @@ objects: secretKeyRef: name: aws key: s3-secret-access-key + - name: GEIMINI_SA_CLIENT_ID + valueFrom: + secretKeyRef: + name: gemini-server + key: gemini-sa-client-id + - name: GEMINI_SA_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: gemini-server + key: gemini-sa-client-secret + - name: AUTH_SERVICE_HOST + valueFrom: + configMapKeyRef: + name: bayesian-config + key: auth-url - name: GEMINI_API_SERVICE_PORT value: "5000" - name: GEMINI_API_SERVICE_TIMEOUT diff --git a/runtests.sh b/runtests.sh old mode 100755 new mode 100644 index 8e776f1..e12be54 --- a/runtests.sh +++ b/runtests.sh @@ -1,6 +1,6 @@ #!/usr/bin/bash -ex -COVERAGE_THRESHOLD=90 +COVERAGE_THRESHOLD=70 export TERM=xterm TERM=${TERM:-xterm} diff --git a/src/auth.py b/src/auth.py index fe3dd02..916062b 100644 --- a/src/auth.py +++ b/src/auth.py @@ -3,6 +3,7 @@ from flask import current_app, request import jwt +import requests from os import getenv @@ -48,6 +49,39 @@ def get_audiences(): return getenv('BAYESIAN_JWT_AUDIENCE').split(',') +def init_auth_sa_token(): + """Initialize a service token from auth service.""" + auth_server_url = getenv('AUTH_SERVICE_HOST', 'https://auth.prod-preview.openshift.io') + endpoint = '{url}/api/token'.format(url=auth_server_url) + + client_id = getenv('GEIMINI_SA_CLIENT_ID', 'id') + client_secret = getenv('GEMINI_SA_CLIENT_SECRET', 'secret') + + payload = {"grant_type": "client_credentials", + "client_id": client_id.strip(), + "client_secret": client_secret.strip()} + try: + print('TOKEN GENERATION: endpoint is %s' % endpoint) + print('TOKEN GENERATION: payload is %r' % payload) + resp = requests.post(endpoint, json=payload) + print("RESPONSE STATUS = %d" % resp.status_code) + except requests.exceptions.RequestException as e: + raise e + + if resp.status_code == 200: + data = resp.json() + try: + access_token = data['access_token'] + print("Access token has been generated successfully") + except IndexError as e: + print("requests.exceptions.RequestException during Access token generation") + raise requests.exceptions.RequestException + return access_token + else: + print("Unexpected HTTP response. Raised requests.exceptions.RequestException") + raise requests.exceptions.RequestException + + def login_required(view): # pragma: no cover """Check if the login is required and if the user can be authorized.""" # NOTE: the actual authentication 401 failures are commented out for now and will be diff --git a/src/rest_api.py b/src/rest_api.py index f97b1f2..8c48761 100644 --- a/src/rest_api.py +++ b/src/rest_api.py @@ -1,10 +1,12 @@ """Definition of the routes for gemini server.""" import flask +import requests from flask import Flask, request from flask_cors import CORS -from utils import DatabaseIngestion, scan_repo, validate_request_data, retrieve_worker_result +from utils import DatabaseIngestion, scan_repo, validate_request_data,\ + retrieve_worker_result, alert_user from f8a_worker.setup_celery import init_selinon -from auth import login_required +from auth import login_required, init_auth_sa_token from exceptions import HTTPError app = Flask(__name__) @@ -12,6 +14,13 @@ init_selinon() +SERVICE_TOKEN = 'token' +try: + SERVICE_TOKEN = init_auth_sa_token() +except requests.exceptions.RequestException as e: + print('Unable to set authentication token for internal service calls. {}' + .format(e)) + @app.route('/api/v1/readiness') def readiness(): @@ -60,7 +69,7 @@ def register(): try: # First time ingestion DatabaseIngestion.store_record(input_json) - status = scan_repo(input_json) + status = scan_repo(input_json, SERVICE_TOKEN) if status is not True: resp_dict["success"] = False resp_dict["summary"] = "New Repo Scan Initialization Failure" @@ -135,6 +144,123 @@ def report(): return flask.jsonify(response), 404 +@app.route('/api/v1/user-repo/scan', methods=['POST']) +@login_required +def user_repo_scan(): + """ + Endpoint for scanning an OSIO user's repository. + + Runs a scan to find out security vulnerability in a user's repository + """ + resp_dict = { + "status": "success", + "summary": "" + } + + if request.content_type != 'application/json': + resp_dict["status"] = "failure" + resp_dict["summary"] = "Set content type to application/json" + return flask.jsonify(resp_dict), 400 + + input_json = request.get_json() + + # Return a dummy response for the endpoint while the development is in progress + if 'dev' not in input_json: + return flask.jsonify({'summary': 'Repository scan initiated'}), 200 + + validate_string = "{} cannot be empty" + if 'git-url' not in input_json: + validate_string = validate_string.format("git-url") + return False, validate_string + + # Call the worker flow to run a user repository scan asynchronously + status = alert_user(input_json, SERVICE_TOKEN) + if status is not True: + resp_dict["status"] = "failure" + resp_dict["summary"] = "Scan initialization failure" + return flask.jsonify(resp_dict), 500 + + resp_dict.update({ + "summary": "Report for {} is being generated in the background. You will " + "be notified via your preferred openshift.io notification mechanism " + "on its completion.".format(input_json.get('git-url')), + }) + + return flask.jsonify(resp_dict), 200 + + +@app.route('/api/v1/user-repo/notify', methods=['POST']) +@login_required +def notify_user(): + """ + Endpoint for notifying security vulnerability in a repository. + + Runs a scan to find out security vulnerability in a user's repository + """ + resp_dict = { + "status": "success", + "summary": "" + } + + if request.content_type != 'application/json': + resp_dict["status"] = "failure" + resp_dict["summary"] = "Set content type to application/json" + return flask.jsonify(resp_dict), 400 + + input_json = request.get_json() + + # Return a dummy response for the endpoint while the development is in progress + if 'dev' not in input_json: + return flask.jsonify({'summary': 'Notification service called'}), 200 + + validate_string = "{} cannot be empty" + if 'epv_list' not in input_json: + resp_dict["status"] = "failure" + resp_dict["summary"] = "Required parameter 'epv_list' is missing " \ + "in the request" + return flask.jsonify(resp_dict), 400 + + # Call the worker flow to run a user repository scan asynchronously + status = alert_user(input_json, SERVICE_TOKEN, epv_list=input_json['epv_list']) + if status is not True: + resp_dict["status"] = "failure" + resp_dict["summary"] = "Scan initialization failure" + return flask.jsonify(resp_dict), 500 + + resp_dict.update({ + "summary": "Report for {} is being generated in the background. You will " + "be notified via your preferred openshift.io notification mechanism " + "on its completion.".format(input_json.get('git-url')), + }) + + return flask.jsonify(resp_dict), 200 + + +@app.route('/api/v1/user-repo/drop', methods=['POST']) +@login_required +def drop(): + """ + Endpoint to stop monitoring OSIO users' repository. + + Runs a scan to find out security vulnerability in a user's repository + """ + resp_dict = { + "status": "success", + "summary": "" + } + + if request.content_type != 'application/json': + resp_dict["status"] = "failure" + resp_dict["summary"] = "Set content type to application/json" + return flask.jsonify(resp_dict), 400 + + input_json = request.get_json() + + # Return a dummy response for the endpoint while the development is in progress + if 'dev' not in input_json: + return flask.jsonify({'summary': 'Repository scan unsubscribed'}), 200 + + @app.errorhandler(HTTPError) def handle_error(e): # pragma: no cover """Handle http error response.""" diff --git a/src/utils.py b/src/utils.py index fd8a9ba..7b4ccca 100644 --- a/src/utils.py +++ b/src/utils.py @@ -267,6 +267,18 @@ def scan_repo(data): return True +def alert_user(data, service_token="", epv_list=[]): + """Invoke worker flow to scan user repository.""" + args = {'github_repo': data['git-url'], + 'service_token': service_token, + 'email_ids': data.get('email-ids', 'dummy'), + 'epv_list': epv_list} + + d_id = server_run_flow('osioUserNotificationFlow', args) + logger.info("DISPATCHER ID = {}".format(d_id)) + return True + + def fetch_public_key(app): """Get public key and caches it on the app object for future use.""" # TODO: even though saving the key on the app object is not very nice, diff --git a/swagger.yaml b/swagger.yaml index bbbc480..d8128f3 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -82,6 +82,90 @@ paths: description: Request unauthorized '404': description: Data not found + '/user-repo/scan': + post: + tags: + - Scan Services + operationId: f8a_scanner.api_v1.scan + summary: Scan an OSIO user repository. This will be called by the OSIO platform whenever a new repository is added to a space. The client request requires OSIO user token in the authorization header. + consumes: + - application/json + produces: + - application/json + parameters: + - in: body + name: repo + description: repository url + required: true + schema: + $ref: '#/definitions/UserRepoInput' + responses: + '200': + description: Repository scan initiated + '400': + description: Bad request from the client + '401': + description: Request unauthorized + '404': + description: Data not found + '500': + description: Internal server error + '/user-repo/notify': + post: + tags: + - Scan Services + operationId: f8a_scanner.api_v1.notify + summary: Call the notification service with the scan report. + consumes: + - application/json + produces: + - application/json + parameters: + - in: body + name: repo + description: List of ecosystem-package-version + required: true + schema: + $ref: '#/definitions/EPVList' + responses: + '200': + description: Notification service called + '400': + description: Bad request from the client + '401': + description: Request unauthorized + '404': + description: Data not found + '500': + description: Internal server error + '/user-repo/drop': + post: + tags: + - Scan Services + operationId: f8a_scanner.api_v1.drop + summary: Stop monitoring an OSIO user repository. This will be triggered by the platform whenever a codebase is removed from a space. The client request requires OSIO user token in the authorization header. + consumes: + - application/json + produces: + - application/json + parameters: + - in: body + name: repo + description: repository url and email id + required: true + schema: + $ref: '#/definitions/UserRepoInput' + responses: + '200': + description: Repository scan unsubscribed + '400': + description: Bad request from the client + '401': + description: Request unauthorized + '404': + description: Data not found + '500': + description: Internal server error definitions: RegisterResponse: title: Response Data for Register Endpoint @@ -141,4 +225,31 @@ definitions: type: string git-sha: type: string - + UserRepoInput: + title: User Repository Scan Inputs + description: Parameters to call user repository scan + properties: + git-url: + type: string + email-ids: + type: array + items: + type: string + EPV: + title: EPV + description: Describes EPV + properties: + ecosystem: + type: string + name: + type: string + version: + type: string + EPVList: + title: User Repository notify inputs + description: Parameters to call user repository notify + properties: + epv_list: + type: array + items: + $ref: '#/definitions/EPV'