From 94b2ca20549c1c34d10f2f053e3560f7e906d372 Mon Sep 17 00:00:00 2001 From: Yusuf Zainee Date: Wed, 13 Feb 2019 19:29:30 +0530 Subject: [PATCH 1/2] python2 to python3 migration --- Dockerfile.data-model | 2 ++ Dockerfile.data-model.rhel | 2 ++ runtests.sh | 4 ++-- sanitycheck.py | 4 ++-- src/cve.py | 6 +++--- src/data_importer.py | 7 +++---- src/data_source/s3_data_source.py | 4 ++-- src/graph_manager.py | 2 +- src/graph_populator.py | 4 ++-- src/rest_api.py | 12 ++++++------ src/utils.py | 2 +- test/conftest.py | 2 +- test/test_cve.py | 8 ++++---- test/test_data_importer.py | 2 +- test/test_graph_manager.py | 4 ++-- test/test_graph_populator.py | 4 ++-- test/test_init.py | 4 ++-- test/test_insertion_from_minio.py | 6 +++--- test/test_postgres_handler.py | 2 +- test/test_rest_api.py | 26 +++++++++++++------------- test/test_s3_data_source.py | 4 ++-- test/test_utils.py | 12 ++++++------ 22 files changed, 63 insertions(+), 60 deletions(-) diff --git a/Dockerfile.data-model b/Dockerfile.data-model index f8b3c050..8cf81c87 100644 --- a/Dockerfile.data-model +++ b/Dockerfile.data-model @@ -22,6 +22,8 @@ RUN yum install -y epel-release && \ # Note: cron daemon ( crond ) will be invoked from within entry point # -------------------------------------------------------------------------------------------------------------- +RUN pip3 install git+https://git@github.com/fabric8-analytics/fabric8-analytics-version-comparator.git +RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git COPY ./ /tmp/f8a_data_model/ COPY ./src /src diff --git a/Dockerfile.data-model.rhel b/Dockerfile.data-model.rhel index 6771f670..7175b6ca 100644 --- a/Dockerfile.data-model.rhel +++ b/Dockerfile.data-model.rhel @@ -18,6 +18,8 @@ LABEL author "Devtools " # Note: cron daemon ( crond ) will be invoked from within entry point # -------------------------------------------------------------------------------------------------------------- +RUN pip3 install git+https://git@github.com/fabric8-analytics/fabric8-analytics-version-comparator.git +RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@latest COPY ./ /tmp/f8a_data_model/ COPY ./src /src diff --git a/runtests.sh b/runtests.sh index c6b2c626..0bbb0bf6 100755 --- a/runtests.sh +++ b/runtests.sh @@ -76,7 +76,7 @@ setup_virtualenv source venv/bin/activate -PYTHONPATH=$(pwd)/src +PYTHONPATH=$(pwd) export PYTHONPATH export BAYESIAN_PGBOUNCER_SERVICE_HOST="localhost" @@ -97,7 +97,7 @@ echo "*** Unit tests ***" echo "*****************************************" echo "Check for sanity of the connections..." -if python sanitycheck.py +if python3 sanitycheck.py then python3 populate_schema.py py.test --cov=src/ --cov-report term-missing --cov-fail-under=$COVERAGE_THRESHOLD -vv -s test/ diff --git a/sanitycheck.py b/sanitycheck.py index 8d612e37..23f05e4a 100644 --- a/sanitycheck.py +++ b/sanitycheck.py @@ -1,10 +1,10 @@ """Sanity check of the graph DB REST API.""" -from graph_manager import BayesianGraph +from src.graph_manager import BayesianGraph import time import sys import logging -import config +from src import config logging.basicConfig() logger = logging.getLogger(config.APP_NAME) diff --git a/src/cve.py b/src/cve.py index 2f68cda3..4c75a736 100644 --- a/src/cve.py +++ b/src/cve.py @@ -1,9 +1,9 @@ """This module encapsulates CVE related queries.""" import logging -from graph_populator import GraphPopulator -from graph_manager import BayesianGraph -from utils import get_timestamp, call_gremlin +from src.graph_populator import GraphPopulator +from src.graph_manager import BayesianGraph +from src.utils import get_timestamp, call_gremlin logger = logging.getLogger(__name__) diff --git a/src/data_importer.py b/src/data_importer.py index 8da87251..343bdea6 100644 --- a/src/data_importer.py +++ b/src/data_importer.py @@ -1,12 +1,11 @@ """Module with functions to fetch data from the S3 data source.""" -from graph_populator import GraphPopulator +from src.graph_populator import GraphPopulator import logging -import config -import traceback +from src import config import json import requests -from data_source.s3_data_source import S3DataSource +from src.data_source.s3_data_source import S3DataSource from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker diff --git a/src/data_source/s3_data_source.py b/src/data_source/s3_data_source.py index e5cdc843..0414b5ed 100644 --- a/src/data_source/s3_data_source.py +++ b/src/data_source/s3_data_source.py @@ -1,10 +1,10 @@ """Data source that returns data read from the AWS S3 database.""" -from data_source.abstract_data_source import AbstractDataSource +from src.data_source.abstract_data_source import AbstractDataSource import botocore import boto3 import json -import config +from src import config class S3DataSource(AbstractDataSource): diff --git a/src/graph_manager.py b/src/graph_manager.py index 21ae1a94..5b094029 100644 --- a/src/graph_manager.py +++ b/src/graph_manager.py @@ -1,6 +1,6 @@ """Template for a singleton object which will have reference to Graph object.""" -import config +from src import config import json import requests import os diff --git a/src/graph_populator.py b/src/graph_populator.py index 7b27c643..0488c820 100644 --- a/src/graph_populator.py +++ b/src/graph_populator.py @@ -5,8 +5,8 @@ import time from dateutil.parser import parse as parse_datetime from six import string_types -import config -from utils import get_current_version +from src import config +from src.utils import get_current_version from datetime import datetime from f8a_utils.versions import get_latest_versions_for_ep diff --git a/src/rest_api.py b/src/rest_api.py index b57e1910..c1f29c3c 100644 --- a/src/rest_api.py +++ b/src/rest_api.py @@ -6,12 +6,12 @@ from flask_cors import CORS import json import sys -import data_importer -from graph_manager import BayesianGraph -from graph_populator import GraphPopulator -from cve import CVEPut, CVEDelete, CVEGet, CVEDBVersion +from src import data_importer +from src.graph_manager import BayesianGraph +from src.graph_populator import GraphPopulator +from src.cve import CVEPut, CVEDelete, CVEGet, CVEDBVersion from raven.contrib.flask import Sentry -import config +from src import config as config from werkzeug.contrib.fixers import ProxyFix import logging from flask import Blueprint, current_app @@ -313,7 +313,7 @@ def cvedb_version_put(): def create_app(): """Create Flask app object.""" new_app = Flask(config.APP_NAME) - new_app.config.from_object('config') + new_app.config.from_object('src.config') CORS(new_app) new_app.register_blueprint(api_v1) return new_app diff --git a/src/utils.py b/src/utils.py index 5e038c13..87baf57c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -5,7 +5,7 @@ import requests import json import logging -import config +from src import config from datetime import datetime logger = logging.getLogger(__name__) diff --git a/test/conftest.py b/test/conftest.py index 217393ec..74c18400 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,7 +2,7 @@ import pytest -from rest_api import create_app +from src.rest_api import create_app @pytest.fixture(scope='session') diff --git a/test/test_cve.py b/test/test_cve.py index 4a27ffcd..ca9cbd3b 100644 --- a/test/test_cve.py +++ b/test/test_cve.py @@ -3,7 +3,7 @@ import pytest from mock import patch -from cve import ( +from src.cve import ( CVEPut, CVEDelete, CVEGet, cve_node_replace_script_template, cve_node_delete_script_template @@ -123,7 +123,7 @@ def test_cve_delete_prepare_payload(): assert bindings['cve_id'] -@patch("cve.call_gremlin") +@patch("src.cve.call_gremlin") def test_cve_get_e(mocker): """Test getting CVEs for (ecosystem).""" mocker.return_value = {'result': {'data': ['CVE-2018-0001']}} @@ -139,7 +139,7 @@ def test_cve_get_e(mocker): assert response['cve_ids'][0] == 'CVE-2018-0001' -@patch("cve.call_gremlin") +@patch("src.cve.call_gremlin") def test_cve_get_ep(mocker): """Test getting CVEs for (ecosystem,name).""" mocker.return_value = {'result': {'data': ['CVE-2018-0001', 'CVE-2018-0002']}} @@ -156,7 +156,7 @@ def test_cve_get_ep(mocker): assert response['cve_ids'][1] in ('CVE-2018-0001', 'CVE-2018-0002') -@patch("cve.call_gremlin") +@patch("src.cve.call_gremlin") def test_cve_get_epv(mocker): """Test getting CVEs for (ecosystem,name,version).""" mocker.return_value = {'result': {'data': []}} diff --git a/test/test_data_importer.py b/test/test_data_importer.py index 05e2a1ec..2d4e83bb 100644 --- a/test/test_data_importer.py +++ b/test/test_data_importer.py @@ -1,6 +1,6 @@ """Tests for the data_importer module (to be done).""" -import data_importer +from src import data_importer def test_parse_int_or_none_for_integer_input(): diff --git a/test/test_graph_manager.py b/test/test_graph_manager.py index f8ed3f0e..75c9e453 100644 --- a/test/test_graph_manager.py +++ b/test/test_graph_manager.py @@ -1,8 +1,8 @@ """Tests for the graph_manager module (to be done).""" -from graph_manager import BayesianGraph as g +from src.graph_manager import BayesianGraph as g import logging -import config +from src import config logger = logging.getLogger(config.APP_NAME) diff --git a/test/test_graph_populator.py b/test/test_graph_populator.py index ff6ee82e..dcb2b83e 100644 --- a/test/test_graph_populator.py +++ b/test/test_graph_populator.py @@ -1,9 +1,9 @@ """Tests for the graph_populator module.""" -from graph_populator import GraphPopulator +from src.graph_populator import GraphPopulator import pytest import logging -import config +from src import config logger = logging.getLogger(config.APP_NAME) diff --git a/test/test_init.py b/test/test_init.py index d4ea916e..ce7db89f 100644 --- a/test/test_init.py +++ b/test/test_init.py @@ -1,11 +1,11 @@ """Tests for the __init__ script.""" -import __init__ +from src import logger def test_logger(): """Test the logger initialized in __init__.""" - assert __init__.logger is not None + assert logger is not None if __name__ == '__main__': diff --git a/test/test_insertion_from_minio.py b/test/test_insertion_from_minio.py index 99c6d7bb..80f508b5 100644 --- a/test/test_insertion_from_minio.py +++ b/test/test_insertion_from_minio.py @@ -5,12 +5,12 @@ """ import logging -import config +from src import config import traceback from minio import Minio from minio.error import ResponseError, BucketAlreadyOwnedByYou, BucketAlreadyExists -from data_importer import import_epv_from_s3_http -from graph_manager import BayesianGraph +from src.data_importer import import_epv_from_s3_http +from src.graph_manager import BayesianGraph logging.basicConfig() logger = logging.getLogger(__name__) diff --git a/test/test_postgres_handler.py b/test/test_postgres_handler.py index ee666723..c1a42506 100644 --- a/test/test_postgres_handler.py +++ b/test/test_postgres_handler.py @@ -5,7 +5,7 @@ """ import logging -from data_importer import PostgresHandler +from src.data_importer import PostgresHandler logging.basicConfig() logger = logging.getLogger(__name__) diff --git a/test/test_rest_api.py b/test/test_rest_api.py index 208a6024..2ea341f2 100644 --- a/test/test_rest_api.py +++ b/test/test_rest_api.py @@ -1,7 +1,7 @@ """Tests for the rest_api module.""" import logging -import config +from src import config import json from flask import url_for from mock import patch @@ -122,7 +122,7 @@ def test_ingest_to_graph_valid(client): assert data['message'] == 'Invalid keys found in input: ' + ','.join(epv_keys) -@patch("rest_api.data_importer.import_epv_from_s3_http") +@patch("src.rest_api.data_importer.import_epv_from_s3_http") def test_ingest_to_graph_report(mocker, client): """Add test for ingest to graph API when report status is Failure.""" input_data = [ @@ -235,7 +235,7 @@ def test_selective_ingest_valid_source(client): def test_handle_properties_put(client, mocker): """Test PUT on /api/v1//

//properties.""" - gremlin_mock = mocker.patch('rest_api.BayesianGraph.execute') + gremlin_mock = mocker.patch('src.rest_api.BayesianGraph.execute') gremlin_mock.return_value = (True, {}) url = url_for('api_v1.handle_properties', ecosystem='maven', package='net.iharder:base64', version='2.3.9') @@ -262,7 +262,7 @@ def test_handle_properties_put(client, mocker): def test_handle_properties_delete(client, mocker): """Test DELETE on /api/v1//

//properties.""" - gremlin_mock = mocker.patch('rest_api.BayesianGraph.execute') + gremlin_mock = mocker.patch('src.rest_api.BayesianGraph.execute') gremlin_mock.return_value = (True, {}) url = url_for('api_v1.handle_properties', ecosystem='maven', package='net.iharder:base64', version='2.3.9') @@ -334,7 +334,7 @@ def test_create_blank_nodes_valid(client): assert data['epv_nodes_created'] == 1 -@patch("rest_api.data_importer.create_graph_nodes") +@patch("src.rest_api.data_importer.create_graph_nodes") def test_create_blank_nodes_report_status(mocker, client): """Add test to create blank nodes API when report status is Failure.""" input_data = [ @@ -353,7 +353,7 @@ def test_create_blank_nodes_report_status(mocker, client): assert response.status_code == 500 -@patch("rest_api.CVEPut.process") +@patch("src.rest_api.CVEPut.process") def test_cves_put(mocker, client): """Test PUT /api/v1/cves.""" mocker.return_value = {} @@ -368,7 +368,7 @@ def test_cves_put(mocker, client): assert response.json == {} -@patch("rest_api.CVEPut.process") +@patch("src.rest_api.CVEPut.process") def test_cves_put_invalid_input(mocker, client): """Test PUT /api/v1/cves with invalid input.""" mocker.return_value = {} @@ -383,7 +383,7 @@ def test_cves_put_invalid_input(mocker, client): assert 'error' in response.json -@patch("rest_api.CVEDelete.process") +@patch("src.rest_api.CVEDelete.process") def test_cves_delete(mocker, client): """Test DELETE /api/v1/cves.""" mocker.return_value = {} @@ -398,7 +398,7 @@ def test_cves_delete(mocker, client): assert response.json == {} -@patch("rest_api.CVEDelete.process") +@patch("src.rest_api.CVEDelete.process") def test_cves_delete_invalid_input(mocker, client): """Test DELETE /api/v1/cves with invalid input.""" mocker.return_value = {} @@ -413,7 +413,7 @@ def test_cves_delete_invalid_input(mocker, client): assert 'error' in response.json -@patch("rest_api.CVEGet.get") +@patch("src.rest_api.CVEGet.get") def test_cves_get_e(mocker, client): """Test GET /api/v1/cves/.""" mocker.return_value = {'count': 1, 'cve_ids': ['CVE-2018-0001']} @@ -425,7 +425,7 @@ def test_cves_get_e(mocker, client): assert response.status_code == 200 -@patch("rest_api.CVEGet.get") +@patch("src.rest_api.CVEGet.get") def test_cves_get_ep(mocker, client): """Test GET /api/v1/cves//.""" mocker.return_value = {'count': 1, 'cve_ids': ['CVE-2018-0001']} @@ -437,7 +437,7 @@ def test_cves_get_ep(mocker, client): assert response.status_code == 200 -@patch("cve.call_gremlin") +@patch("src.cve.call_gremlin") def test_cvedb_version_get(mocker, client): """Test GET /api/v1/cvedb-version.""" mocker.return_value = {'result': {'data': ['9f4d54dd1a21584a40596c05d60ab00974953047']}} @@ -453,7 +453,7 @@ def test_cvedb_version_get(mocker, client): assert resp['version'] == '9f4d54dd1a21584a40596c05d60ab00974953047' -@patch("cve.call_gremlin") +@patch("src.cve.call_gremlin") def test_cvedb_version_put(mocker, client): """Test PUT /api/v1/cvedb-version.""" mocker.return_value = {'result': {'data': ['9f4d54dd1a21584a40596c05d60ab00974953047']}} diff --git a/test/test_s3_data_source.py b/test/test_s3_data_source.py index b1e8d8e2..314b5c25 100644 --- a/test/test_s3_data_source.py +++ b/test/test_s3_data_source.py @@ -2,8 +2,8 @@ # TODO: to be implemented -from data_source.s3_data_source import S3DataSource -import config +from src.data_source.s3_data_source import S3DataSource +from src import config import pytest diff --git a/test/test_utils.py b/test/test_utils.py index 4bfbc68c..120e768d 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -2,9 +2,9 @@ import pytest import datetime -from utils import get_current_version, execute_gremlin_dsl, get_timestamp, call_gremlin +from src.utils import get_current_version, execute_gremlin_dsl, get_timestamp, call_gremlin import logging -import config +from src import config from mock import patch from conftest import RequestsMockResponse @@ -18,7 +18,7 @@ def test_get_current_version(): assert out2 == -1 -@patch("utils.get_session_retry") +@patch("src.utils.get_session_retry") def test_execute_gremlin_dsl(mocker): """Test the function get_version_information.""" mocker.return_value = "" @@ -60,7 +60,7 @@ def status_code(self): return '404' -@patch("utils.get_session_retry") +@patch("src.utils.get_session_retry") def test_execute_gremlin_dsl2(mocker): """Test the function get_version_information.""" mocker.return_value = MockedSession() @@ -84,14 +84,14 @@ def test_get_timestamp(): assert result == timestamp -@patch("utils.requests.post") +@patch("src.utils.requests.post") def test_gremlin_call(mocker): """Test utils.call_gremlin().""" mocker.return_value = RequestsMockResponse({}, 200) assert call_gremlin({'dummy': 'payload'}) == {} -@patch("utils.requests.post") +@patch("src.utils.requests.post") def test_bad_gremlin_call(mocker): """Test utils.call_gremlin().""" mocker.return_value = RequestsMockResponse({}, 500) From 40f4e2fafa83d0fb4284d42a9339281060663f60 Mon Sep 17 00:00:00 2001 From: Yusuf Zainee Date: Fri, 15 Feb 2019 14:57:52 +0530 Subject: [PATCH 2/2] dummy commit. to trigger build --- Dockerfile.data-model | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile.data-model b/Dockerfile.data-model index 8cf81c87..f67d8f14 100644 --- a/Dockerfile.data-model +++ b/Dockerfile.data-model @@ -33,4 +33,3 @@ ADD scripts/entrypoint.sh /bin/entrypoint.sh ADD populate_schema.py /populate_schema.py ENTRYPOINT ["/bin/entrypoint.sh"] -