diff --git a/README.md b/README.md index ad54e13f0..11c7aae74 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ Both the web-interface, and the programmatic-api, have corresponding ### Web Interface The [web-interface](https://github.com/jeff1evesque/machine-learning/blob/master/interface/templates/index.html), - can be accessed within the browser on `https://192.168.99.101:8080`: + can be accessed within the browser on `https://192.168.99.101:8585`: ![web-interface](https://user-images.githubusercontent.com/2907085/39499223-97b96fce-4d7a-11e8-96e2-c4e31f6b8e09.JPG 'web-interface') @@ -135,7 +135,7 @@ A post request, can be implemented in python, as follows: ```python import requests -endpoint = 'https://192.168.99.101:9090/load-data' +endpoint = 'https://192.168.99.101:9595/load-data' headers = { 'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json' @@ -151,6 +151,6 @@ requests.post(endpoint, headers=headers, data=json_string_here) It is important to remember that the [`docker-compose.development.yml`](https://github.com/jeff1evesque/machine-learning/blob/3889788a8343a4b7cef2cf84166f9bd35d83021c/docker-compose.development.yml#L33-L43), has defined two port forwards, each assigned to its corresponding reverse - proxy. This allows port `8080` on the host, to map into the `webserver-web` - container. A similar case for the programmatic-api, uses port `9090` on the + proxy. This allows port `8585` on the host, to map into the `webserver-web` + container. A similar case for the programmatic-api, uses port `9595` on the host. diff --git a/brain/cache/hset.py b/brain/cache/hset.py index fee199099..51e813815 100644 --- a/brain/cache/hset.py +++ b/brain/cache/hset.py @@ -9,15 +9,13 @@ from brain.cache.query import Query -class Hset(object): +class Hset: ''' This class provides an interface to cache, and uncache the redis hash data structure. Specifically, necessary data components is passed into the corresponding class method. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): @@ -36,7 +34,7 @@ def __init__(self): # start redis client try: self.myRedis.start_redis() - except Exception, error: + except Exception as error: self.list_error.append(str(error)) def cache(self, hash_name, key, value): @@ -48,9 +46,9 @@ def cache(self, hash_name, key, value): try: self.myRedis.hset(hash_name, key, value) - except Exception, error: + except Exception as error: self.list_error.append(str(error)) - print self.list_error + print(self.list_error) def uncache(self, hash_name, key): ''' @@ -61,6 +59,6 @@ def uncache(self, hash_name, key): try: return {'result': self.myRedis.hget(hash_name, key), 'error': None} - except Exception, error: + except Exception as error: self.list_error.append(str(error)) return {'result': None, 'error': self.list_error} diff --git a/brain/cache/model.py b/brain/cache/model.py index 0aeec757c..f5a9fdecd 100644 --- a/brain/cache/model.py +++ b/brain/cache/model.py @@ -10,7 +10,7 @@ from brain.converter.model import Model as Converter -class Model(object): +class Model: ''' This class provides an interface to cache, and uncache the redis hash @@ -18,8 +18,6 @@ class Model(object): corresponding class method, which allow computed model(s) to be stored into a NoSQL datastore. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self, model=None): @@ -39,7 +37,7 @@ def __init__(self, model=None): # start redis client try: self.myRedis.start_redis() - except Exception, error: + except Exception as error: self.list_error.append(str(error)) def cache(self, hash_name, key): @@ -53,9 +51,9 @@ def cache(self, hash_name, key): try: serialized = Converter(self.model).serialize() self.myRedis.hset(hash_name, key, serialized) - except Exception, error: + except Exception as error: self.list_error.append(str(error)) - print self.list_error + print(self.list_error) def uncache(self, hash_name, key): ''' @@ -96,6 +94,6 @@ def get_all_titles(self, name): 'error': 'no previous model found in cache' } - except Exception, error: + except Exception as error: self.list_error.append(str(error)) return {'result': None, 'error': self.list_error} diff --git a/brain/cache/query.py b/brain/cache/query.py index 9d83f1a8f..4446369b9 100644 --- a/brain/cache/query.py +++ b/brain/cache/query.py @@ -10,7 +10,7 @@ from brain.cache.settings import Settings -class Query(object): +class Query: ''' This class provides an interface to various Redis data structures. @@ -32,8 +32,6 @@ class Query(object): concept to 'lists'. Also, the above included redis data structures, provides enough flexibility to accomplish most requirements. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self, db_num=0, host=None, port=None): diff --git a/brain/cache/settings.py b/brain/cache/settings.py index 2c2dad0c3..c686cfe58 100644 --- a/brain/cache/settings.py +++ b/brain/cache/settings.py @@ -9,14 +9,12 @@ from flask import current_app -class Settings(object): +class Settings: ''' This class provides an interface to get, or set the redis host, or redis port. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): diff --git a/brain/converter/model.py b/brain/converter/model.py index ed99e9540..abf908438 100644 --- a/brain/converter/model.py +++ b/brain/converter/model.py @@ -10,14 +10,12 @@ from sklearn import svm, preprocessing -class Model(object): +class Model: ''' This class provides an interface to serialize, and deserialize an SVM object. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self, model): diff --git a/brain/converter/operation.py b/brain/converter/operation.py index c9d727099..3820db338 100644 --- a/brain/converter/operation.py +++ b/brain/converter/operation.py @@ -9,15 +9,13 @@ ''' -class Operation(object): +class Operation: ''' This class provides an interface to restructure the supplied data into a consistent structure, which allows successive parsers to implement corresponding logic. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self, settings, dataset=None): diff --git a/brain/converter/settings.py b/brain/converter/settings.py index 6d27ca280..bd02345e5 100644 --- a/brain/converter/settings.py +++ b/brain/converter/settings.py @@ -9,15 +9,13 @@ ''' -class Settings(object): +class Settings: ''' This class provides an interface to restructure the supplied data into a consistent structure, which allows successive parsers to implement corresponding logic. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self, settings, dataset=None): diff --git a/brain/database/account.py b/brain/database/account.py index dd0e6010e..0b551d4a3 100644 --- a/brain/database/account.py +++ b/brain/database/account.py @@ -10,13 +10,11 @@ from brain.database.query import SQL -class Account(object): +class Account: ''' This class provides an interface to the users account. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): @@ -71,7 +69,7 @@ def check_username(self, username): sql_statement = 'SELECT * '\ 'FROM tbl_user '\ 'WHERE username=%s' - args = (username) + args = (username,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) @@ -95,7 +93,7 @@ def check_email(self, email): sql_statement = 'SELECT * '\ 'FROM tbl_user '\ 'WHERE email=%s' - args = (email) + args = (email,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) @@ -119,7 +117,7 @@ def get_password(self, username): sql_statement = 'SELECT password '\ 'FROM tbl_user '\ 'WHERE username=%s' - args = (username) + args = (username,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) @@ -143,7 +141,7 @@ def get_uid(self, username): sql_statement = 'SELECT id_user '\ 'FROM tbl_user '\ 'WHERE username=%s' - args = (username) + args = (username,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) diff --git a/brain/database/dataset.py b/brain/database/dataset.py index a6291036b..3c5f86c27 100644 --- a/brain/database/dataset.py +++ b/brain/database/dataset.py @@ -10,7 +10,7 @@ from brain.database.query import NoSQL -class Collection(object): +class Collection: ''' This class provides an interface to retrieve, and store various parameters, @@ -18,8 +18,6 @@ class Collection(object): Note: this class is invoked within 'base_data.py' - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): @@ -55,6 +53,12 @@ def query(self, collection, operation, payload=None): if operation == 'drop_collection': self.nosql.connect() response = self.nosql.execute(operation, collection) + + elif operation == 'count_documents': + self.nosql.connect(collection) + response = self.nosql.execute(operation, payload) + response['result'] = int(response['result'] or 0) + else: self.nosql.connect(collection) response = self.nosql.execute(operation, payload) diff --git a/brain/database/entity.py b/brain/database/entity.py index baef961ef..a9acdbb2f 100644 --- a/brain/database/entity.py +++ b/brain/database/entity.py @@ -10,7 +10,7 @@ from brain.database.query import SQL -class Entity(object): +class Entity: ''' This class provides an interface to save, retrieve an SVM entity title, @@ -19,8 +19,6 @@ class Entity(object): Note: this class is invoked within 'model_generate.py', 'base_data.py', and 'data_append.py' - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self, premodel_data=None, session_type=None): @@ -106,7 +104,7 @@ def get_title(self, id_entity): sql_statement = 'SELECT title '\ 'FROM tbl_dataset_entity '\ 'WHERE id_entity=%s' - args = (id_entity) + args = (id_entity,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) @@ -135,7 +133,7 @@ def get_collections(self, uid): 'FROM tbl_dataset_entity '\ 'WHERE uid_created=%s '\ 'ORDER BY datetime_created' - args = (uid) + args = (uid,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) @@ -166,7 +164,7 @@ def get_collection_count(self, uid): 'FROM tbl_dataset_entity '\ 'WHERE uid_created=%s'\ ') AS c' - args = (uid) + args = (uid,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) @@ -176,7 +174,7 @@ def get_collection_count(self, uid): if response_error: return {'error': response_error, 'result': None} else: - return {'error': None, 'result': int(response['result'][0][0])} + return {'error': None, 'result': int(response['result'][0][0] or 0)} def remove_entity(self, uid, collection): ''' diff --git a/brain/database/model_type.py b/brain/database/model_type.py index 3af415b4e..e0009cd23 100644 --- a/brain/database/model_type.py +++ b/brain/database/model_type.py @@ -10,7 +10,7 @@ from brain.database.query import SQL -class ModelType(object): +class ModelType: ''' Note: this class explicitly inherits the 'new-style' class. @@ -20,7 +20,7 @@ class ModelType(object): def __init__(self): ''' - This constructor is responsible for defining class variables. + This class provides and interface between the model_type. ''' @@ -49,7 +49,7 @@ def get_model_type(self, collection): ' INNER JOIN tbl_model_type mtype'\ ' ON mid.model_type = mtype.id_model'\ ' WHERE mid.collection=%s' - args = (collection) + args = (collection,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) diff --git a/brain/database/prediction.py b/brain/database/prediction.py index 47889b6f0..e132a3672 100644 --- a/brain/database/prediction.py +++ b/brain/database/prediction.py @@ -10,14 +10,12 @@ from brain.database.query import SQL -class Prediction(object): +class Prediction: ''' This class provides an interface to save, or retrieve, a previously generated svm or, svr prediction result. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): @@ -79,14 +77,14 @@ def save(self, data, model_type, title): sql_statement = 'INSERT INTO tbl_svm_results_probability '\ '(id_result, probability) VALUES(%s, %s)' args = (svm_results['id'], x) - self.sql.execute('insert', sql_statement, args,) + self.sql.execute('insert', sql_statement, args) # svm decision function for x in decision_function: sql_statement = 'INSERT INTO tbl_svm_results_decision_function '\ '(id_result, decision_function) VALUES(%s, %s)' - args = (svm_results['id'], x,) - self.sql.execute('insert', sql_statement, args,) + args = (svm_results['id'], x) + self.sql.execute('insert', sql_statement, args) elif model_type == 'svr': # svr results @@ -94,13 +92,13 @@ def save(self, data, model_type, title): '(model_type, title, result, uid_created, datetime_created) '\ 'VALUES(%s, %s, %s, %s, UTC_TIMESTAMP())' args = (self.model_list.index(model_type) + 1, title, result, self.uid) - svr_results = self.sql.execute('insert', sql_statement, args,) + svr_results = self.sql.execute('insert', sql_statement, args) # svr r2 sql_statement = 'INSERT INTO tbl_svr_results_r2 '\ '(id_result, r2) VALUES(%s, %s)' args = (svr_results['id'], data['r2']) - self.sql.execute('insert', sql_statement, args,) + self.sql.execute('insert', sql_statement, args) # retrieve any error(s) response_error = self.sql.get_errors() @@ -139,7 +137,7 @@ def get_all_titles(self, model_type=None): sql_statement = 'SELECT id_result, title, datetime_created '\ 'FROM tbl_prediction_results '\ 'WHERE uid_created=%s' - args = (self.uid) + args = (self.uid,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) diff --git a/brain/database/query.py b/brain/database/query.py index df52b4162..ce72e6301 100644 --- a/brain/database/query.py +++ b/brain/database/query.py @@ -66,14 +66,12 @@ def get_mongodb(): return g.mongodb -class NoSQL(object): +class NoSQL: ''' This class provides an interface to connect, execute commands, and disconnect from a NoSQL database. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): @@ -108,7 +106,7 @@ def connect(self, collection=None): 'error': None, } - except errors, error: + except Exception as error: self.proceed = False self.list_error.append(error) @@ -168,7 +166,7 @@ def execute(self, operation, payload): elif operation == 'drop_collection': result = self.database.drop_collection(payload) - except errors, error: + except Exception as error: self.list_error.append(error) return { @@ -197,7 +195,7 @@ def disconnect(self): 'error': None, } - except errors, error: + except Exception as error: self.list_error.append(error) return { @@ -215,14 +213,12 @@ def get_errors(self): return self.list_error -class SQL(object): +class SQL: ''' This class provides an interface to connect, execute commands, and disconnect from a SQL database. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self, host=None, user=None, passwd=None): @@ -300,7 +296,7 @@ def execute(self, operation, statement, sql_args=None): 'result': result, } - except MariaClient.Error, error: + except MariaClient.Error as error: self.conn.rollback() self.list_error.append(error) @@ -329,7 +325,7 @@ def disconnect(self): 'id': self.cursor.lastrowid, } - except MariaClient.Error, error: + except MariaClient.Error as error: self.list_error.append(error) return { diff --git a/brain/database/session.py b/brain/database/session.py index f8b6ed56b..717750356 100644 --- a/brain/database/session.py +++ b/brain/database/session.py @@ -10,7 +10,7 @@ from brain.database.query import SQL -class Session(object): +class Session: ''' This class provides an interface to retrieve the 'id_entity', and @@ -18,8 +18,6 @@ class Session(object): Note: this class is invoked within 'views.py' - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): @@ -44,7 +42,7 @@ def get_session_id(self, collection): self.sql.connect(self.db_ml) sql_statement = 'SELECT id_entity FROM tbl_dataset_entity '\ 'WHERE collection=%s' - args = (collection) + args = (collection,) response = self.sql.execute('select', sql_statement, args) # retrieve any error(s) diff --git a/brain/database/settings.py b/brain/database/settings.py index 3105529f2..f7f114ea8 100644 --- a/brain/database/settings.py +++ b/brain/database/settings.py @@ -9,7 +9,7 @@ from flask import current_app -class Database(object): +class Database: ''' This class provides an interface to get, or set the following database @@ -33,9 +33,7 @@ class Database(object): -> DATABASES ON *.* TO 'authenticated'@'localhost'; MariaDB> FLUSH PRIVILEGES; - Note: this class is invoked within 'query.py' - - Note: this class explicitly inherits the 'new-style' class. + Note: this class is invoked within 'query.py'. ''' diff --git a/brain/load_data.py b/brain/load_data.py index 48d47d451..613cb0034 100644 --- a/brain/load_data.py +++ b/brain/load_data.py @@ -16,7 +16,7 @@ from brain.database.session import Session -class Load_Data(object): +class Load_Data: ''' This class provides an interface to load the necessary parameters: @@ -26,8 +26,6 @@ class Load_Data(object): from the SQL database. - generate a prediction using a previous cached model. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self, data, uid=None): diff --git a/brain/session/base.py b/brain/session/base.py index e0179a99a..66cb1386c 100644 --- a/brain/session/base.py +++ b/brain/session/base.py @@ -14,7 +14,7 @@ from brain.validator.settings import Validator -class Base(object): +class Base: ''' This class provides a general base class, used for the following sessions, @@ -25,9 +25,7 @@ class Base(object): - model_generate - model_predict - Note: this class is invoked within 'data_xx.py', 'model_xx.py' - - Note: this class explicitly inherits the 'new-style' class. + Note: this class is invoked within 'data_xx.py', 'model_xx.py'. ''' diff --git a/brain/session/base_data.py b/brain/session/base_data.py index fb55e6d4b..3afe13c04 100644 --- a/brain/session/base_data.py +++ b/brain/session/base_data.py @@ -71,7 +71,7 @@ def validate_id(self, session_id): try: if not int(session_id) > 0: self.list_error.append(error) - except Exception, error: + except Exception as error: self.list_error.append(str(error)) def save_premodel_dataset(self): @@ -97,6 +97,7 @@ def save_premodel_dataset(self): if ( not self.uid and collection_count and + collection_count['result'] and collection_count['result'] >= self.max_collection and collection_adjusted ): @@ -129,6 +130,21 @@ def save_premodel_dataset(self): document ) + current_utc = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") + self.premodel_data['properties']['datetime_saved'] = current_utc + + if self.dataset: + document = { + 'properties': self.premodel_data['properties'], + 'dataset': self.dataset + } + + response = cursor.query( + collection_adjusted, + 'insert_one', + document + ) + else: response = None diff --git a/brain/session/data_append.py b/brain/session/data_append.py index b23e26e23..253abadb9 100644 --- a/brain/session/data_append.py +++ b/brain/session/data_append.py @@ -77,7 +77,7 @@ def save_entity(self, session_type, session_id): 'id_entity': session_id, } - # store entity values in database + # store entity values if ( collection_adjusted and collection_count and diff --git a/brain/session/data_new.py b/brain/session/data_new.py index 51af82b83..fc9aec260 100644 --- a/brain/session/data_new.py +++ b/brain/session/data_new.py @@ -79,7 +79,7 @@ def save_entity(self, session_type, id_entity=None): 'uid': self.uid, } - # store entity values in database + # store entity values if ( collection_adjusted and collection_count and diff --git a/brain/validator/dataset.py b/brain/validator/dataset.py index f3c59a2be..9db0929ab 100644 --- a/brain/validator/dataset.py +++ b/brain/validator/dataset.py @@ -6,18 +6,17 @@ ''' -from voluptuous import Schema, Required, All, Any, Length +from voluptuous import Schema, Required, All, Any, Length, Coerce from voluptuous.humanize import validate_with_humanized_errors +from six import string_types -class Validator(object): +class Validator: ''' This class provides an interface to validate provided dataset(s) during 'data_new', and 'data_append' sessions. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): @@ -38,17 +37,18 @@ def validate_classification(self, data): current_errors = [] schema = Schema({ - Required('dependent-variable'): All(unicode, Length(min=1)), + Required('dependent-variable'): All(string_types[0], Length(min=1)), Required('independent-variables'): [{ - Required(All(unicode, Length(min=1))): Any(int, float), + Required(All(string_types[0], Length(min=1))): Coerce(float), }], + Required('error'): Any(None, string_types[0]), }) for instance in data: try: validate_with_humanized_errors(instance, schema) - except Exception, error: + except Exception as error: split_error = str(error).splitlines() current_errors.extend(split_error) self.list_error.extend(split_error) @@ -67,17 +67,18 @@ def validate_regression(self, data): current_errors = [] schema = Schema({ - Required('dependent-variable'): Any(int, float), + Required('dependent-variable'): Coerce(float), Required('independent-variables'): [{ - Required(All(unicode, Length(min=1))): Any(int, float), + Required(All(string_types[0], Length(min=1))): Coerce(float), }], + Required('error'): Any(None, string_types[0]), }) for instance in data: try: validate_with_humanized_errors(instance, schema) - except Exception, error: + except Exception as error: split_error = str(error).splitlines() current_errors.extend(split_error) self.list_error.extend(split_error) @@ -97,7 +98,7 @@ def validate_value(self, data): try: return float(data) - except Exception, error: + except Exception as error: self.list_error.append(error) return error diff --git a/brain/validator/settings.py b/brain/validator/settings.py index a042c4053..e71f5ac9a 100644 --- a/brain/validator/settings.py +++ b/brain/validator/settings.py @@ -9,16 +9,15 @@ from flask import current_app from voluptuous import Schema, Required, Optional, All, Any, Coerce, In, Length from voluptuous.humanize import validate_with_humanized_errors +from six import string_types -class Validator(object): +class Validator: ''' This class provides an interface to validate the settings for each session. - Note: this class explicitly inherits the 'new-style' class. - ''' def __init__(self): @@ -50,17 +49,17 @@ def validate_settings(self, premodel_settings, session_type): if session_type in ['data_new', 'data_append']: if premodel_settings['stream'] == 'True' or session_type == 'data_new': schema = Schema({ - Required('collection'): All(unicode, Length(min=1)), + Required('collection'): All(string_types[0], Length(min=1)), Required('dataset_type'): In(dataset_type), Required('model_type'): In(model_type), Required('session_type'): Any('data_new', 'data_append'), - Required('session_name'): All(unicode, Length(min=1)), + Required('session_name'): All(string_types[0], Length(min=1)), Optional('stream'): Any('True', 'False'), }) else: schema = Schema({ - Required('collection'): All(unicode, Length(min=1)), + Required('collection'): All(string_types[0], Length(min=1)), Required('dataset_type'): In(dataset_type), Required('model_type'): In(model_type), Required('session_type'): Any('data_new', 'data_append'), @@ -70,7 +69,7 @@ def validate_settings(self, premodel_settings, session_type): # validation on 'model_generate' session if session_type == 'model_generate': schema = Schema({ - Required('collection'): All(unicode, Length(min=1)), + Required('collection'): All(string_types[0], Length(min=1)), Required('model_type'): In(model_type), Required('session_type'): 'model_generate', Optional('stream'): Any('True', 'False'), @@ -82,7 +81,7 @@ def validate_settings(self, premodel_settings, session_type): # validation on 'model_predict' session elif session_type == 'model_predict': schema = Schema({ - Required('collection'): All(unicode, Length(min=1)), + Required('collection'): All(string_types[0], Length(min=1)), Optional('stream'): Any('True', 'False'), Required('prediction_input[]'): [ Any(Coerce(int), Coerce(float)), @@ -93,7 +92,7 @@ def validate_settings(self, premodel_settings, session_type): try: validate_with_humanized_errors(premodel_settings, schema) - except Exception, error: + except Exception as error: split_error = str(error).splitlines() self.list_error.append(split_error) return split_error diff --git a/dockerfile/base.dockerfile b/dockerfile/base.dockerfile index b0afc42ab..a2172fe9c 100644 --- a/dockerfile/base.dockerfile +++ b/dockerfile/base.dockerfile @@ -1,31 +1,29 @@ FROM ubuntu:16.04 ## environment variables -ENV ENVIRONMENT docker -ENV PUPPET /opt/puppetlabs/bin/puppet -ENV ROOT_PROJECT /var/machine-learning -ENV ROOT_PUPPET /etc/puppetlabs +ENV ENVIRONMENT=docker\ + PUPPET=/opt/puppetlabs/bin/puppet\ + ROOT_PROJECT=/var/machine-learning\ + ROOT_PUPPET=/etc/puppetlabs\ + PUPPET_VERSION=puppet5-release-xenial.deb +ENV MODULES=$ROOT_PUPPET/code/modules\ + CONTRIB_MODULES=$ROOT_PUPPET/code/modules_contrib\ + ENVPATH=$ROOT_PUPPET/code/environment/$ENVIRONMENT ## ensure directory RUN mkdir -p $ROOT_PROJECT/log $ROOT_PUPPET/code/environment/$ENVIRONMENT $ROOT_PUPPET/puppet/hiera $ROOT_PUPPET/code/modules_contrib -## environment variables -ENV MODULES $ROOT_PUPPET/code/modules -ENV CONTRIB_MODULES $ROOT_PUPPET/code/modules_contrib -ENV ENVPATH $ROOT_PUPPET/code/environment/$ENVIRONMENT - -## install git, wget +## +## install git, wget, puppet ## ## Note: r10k requires 'git' installed ## RUN apt-get -y update -RUN apt-get -y install git=1:1.9.1* wget=1.16* - -## install puppet -RUN wget https://apt.puppetlabs.com/puppetlabs-release-pc1-trusty.deb -RUN dpkg -i puppetlabs-release-pc1-trusty.deb +RUN apt-get -y install wget=1.17* curl=7.47* +RUN wget https://apt.puppetlabs.com/$PUPPET_VERSION +RUN dpkg -i $PUPPET_VERSION RUN apt-get -y update -RUN apt-get -y install puppet-agent +RUN apt-get -y install git=1:2.7.4-0ubuntu* puppet-agent=5.5* ## copy configs COPY puppet/environment/$ENVIRONMENT/Puppetfile $ENVPATH @@ -39,13 +37,11 @@ COPY hiera.yaml $ROOT_PUPPET/puppet ## ## https://github.com/jeff1evesque/machine-learning/issues/2991 ## -RUN apt-get -y install rubygems-integration=1.5 -RUN gem install semantic_puppet -v 0.1.0 -RUN gem install puppet_forge -v 2.2.5 -RUN gem install r10k -v 2.5.5 +RUN apt-get -y install ruby=1:2.3* rubygems-integration=1.10 +RUN gem install semantic_puppet:0.1.0 puppet_forge:2.2.9 r10k:3.0.2 ## install puppet modules using puppetfile with r10k -RUN PUPPETFILE=$ENVPATH/Puppetfile PUPPETFILE_DIR=$CONTRIB_MODULES r10k puppetfile install +RUN r10k puppetfile install --puppetfile $ENVPATH/Puppetfile --moduledir $CONTRIB_MODULES ## provision with puppet RUN $PUPPET apply apply -e 'class { system: }' --modulepath=$CONTRIB_MODULES:$MODULES --confdir=$ROOT_PUPPET/puppet diff --git a/dockerfile/mariadb.dockerfile b/dockerfile/mariadb.dockerfile index 115c857bf..69d4cf5a6 100644 --- a/dockerfile/mariadb.dockerfile +++ b/dockerfile/mariadb.dockerfile @@ -1,10 +1,10 @@ -FROM jeff1evesque/ml-base:0.7 +FROM jeff1evesque/ml-base:0.8 ## local variables ENV ENVIRONMENT=docker\ PUPPET=/opt/puppetlabs/bin/puppet\ - ROOT_PUPPET=/etc/puppetlabs\ - MODULES=$ROOT_PUPPET/code/modules\ + ROOT_PUPPET=/etc/puppetlabs +ENV MODULES=$ROOT_PUPPET/code/modules\ CONTRIB_MODULES=$ROOT_PUPPET/code/modules_contrib ## copy files into container @@ -16,11 +16,21 @@ RUN apt-get update && $PUPPET apply -e 'class { mariadb: }'\ --modulepath=$CONTRIB_MODULES:$MODULES\ --confdir=$ROOT_PUPPET/puppet +## ## define entrypoint script +## +## @mysqlclient, requires 'apt-get' + 'pip3' dependencies only during runtime: +## +## - https://raspberrypi.stackexchange.com/a/78217. +## RUN printf '#!/bin/bash\n\n\ cd $MODULES/mariadb/scripts\n\ service mysql start\n\ -python setup_tables.py $ROOT_PUPPET/puppet\n\ +apt-get install -y python3-dev libmysqlclient-dev\n\ +pip3 install mysqlclient\n\ +python3 setup_tables.py $ROOT_PUPPET/puppet\n\ +apt-get remove -y python3-dev libmysqlclient-dev\n\ +pip3 uninstall -y mysqlclient\n\ service mysql stop\n\ mysqld\n\ ' > entrypoint diff --git a/dockerfile/mongodb.dockerfile b/dockerfile/mongodb.dockerfile index 7d582f55f..716612ebf 100644 --- a/dockerfile/mongodb.dockerfile +++ b/dockerfile/mongodb.dockerfile @@ -1,11 +1,11 @@ -FROM jeff1evesque/ml-base:0.7 +FROM jeff1evesque/ml-base:0.8 ## local variables -ENV ENVIRONMENT docker -ENV PUPPET /opt/puppetlabs/bin/puppet -ENV ROOT_PUPPET /etc/puppetlabs -ENV MODULES $ROOT_PUPPET/code/modules -ENV CONTRIB_MODULES $ROOT_PUPPET/code/modules_contrib +ENV ENVIRONMENT=docker\ + PUPPET=/opt/puppetlabs/bin/puppet\ + ROOT_PUPPET=/etc/puppetlabs +ENV MODULES=$ROOT_PUPPET/code/modules\ + CONTRIB_MODULES=$ROOT_PUPPET/code/modules_contrib ## copy files into container COPY hiera $ROOT_PUPPET/puppet/hiera diff --git a/dockerfile/nginx.dockerfile b/dockerfile/nginx.dockerfile index 77aff03ae..9cb224c7b 100644 --- a/dockerfile/nginx.dockerfile +++ b/dockerfile/nginx.dockerfile @@ -1,12 +1,13 @@ -FROM jeff1evesque/ml-base:0.7 +FROM jeff1evesque/ml-base:0.8 ## local variables -ENV ENVIRONMENT docker -ENV ROOT_PROJECT /var/machine-learning -ENV PUPPET /opt/puppetlabs/bin/puppet -ENV ROOT_PUPPET /etc/puppetlabs -ENV MODULES $ROOT_PUPPET/code/modules -ENV CONTRIB_MODULES $ROOT_PUPPET/code/modules_contrib +ENV ENVIRONMENT=docker\ + ROOT_PROJECT=/var/machine-learning\ + PUPPET=/opt/puppetlabs/bin/puppet\ + ROOT_PUPPET=/etc/puppetlabs +ENV MODULES=$ROOT_PUPPET/code/modules\ + CONTRIB_MODULES=$ROOT_PUPPET/code/modules_contrib + ARG TYPE ARG VHOST ARG HOST_PORT @@ -21,14 +22,14 @@ COPY puppet/environment/$ENVIRONMENT/modules/reverse_proxy $ROOT_PUPPET/code/mod ## ## provision with puppet: either build a web, or api nginx image. ## -## docker build --build-arg TYPE=api --build-arg RUN=false --build-arg VHOST=machine-learning-api.com --build-arg HOST_PORT=9090 --build-arg LISTEN_PORT=6000 -f nginx.dockerfile -t jeff1evesque/ml-nginx-api:0.7 . -## docker build --build-arg TYPE=web --build-arg RUN=false --build-arg VHOST=machine-learning-web.com --build-arg HOST_PORT=9090 --build-arg LISTEN_PORT=6000 -f nginx.dockerfile -t jeff1evesque/ml-nginx-web:0.7 . +## docker build --build-arg TYPE=api --build-arg RUN=false --build-arg VHOST=machine-learning-api.com --build-arg LISTEN_PORT=6000 -f nginx.dockerfile -t jeff1evesque/ml-nginx-api:0.8 . +## docker build --build-arg TYPE=web --build-arg RUN=false --build-arg VHOST=machine-learning-web.com --build-arg LISTEN_PORT=5000 -f nginx.dockerfile -t jeff1evesque/ml-nginx-web:0.8 . ## -## docker build --build-arg TYPE=api --build-arg RUN=false -f nginx.dockerfile -t jeff1evesque/ml-nginx-api:0.7 . -## docker build --build-arg TYPE=web --build-arg RUN=false -f nginx.dockerfile -t jeff1evesque/ml-nginx-web:0.7 . +## docker build --build-arg TYPE=api --build-arg RUN=false -f nginx.dockerfile -t jeff1evesque/ml-nginx-api:0.8 . +## docker build --build-arg TYPE=web --build-arg RUN=false -f nginx.dockerfile -t jeff1evesque/ml-nginx-web:0.8 . ## -## docker run --hostname nginx-api --name nginx-api -d jeff1evesque/ml-nginx-api:0.7 -## docker run --hostname nginx-web --name nginx-web -d jeff1evesque/ml-nginx-web:0.7 +## docker run --hostname nginx-api --name nginx-api -d jeff1evesque/ml-nginx-api:0.8 +## docker run --hostname nginx-web --name nginx-web -d jeff1evesque/ml-nginx-web:0.8 ## RUN $PUPPET apply -e "class { reverse_proxy: \ run => 'false', \ diff --git a/dockerfile/redis.dockerfile b/dockerfile/redis.dockerfile index 9282a7918..46ebd1ecd 100644 --- a/dockerfile/redis.dockerfile +++ b/dockerfile/redis.dockerfile @@ -1,11 +1,11 @@ -FROM jeff1evesque/ml-base:0.7 +FROM jeff1evesque/ml-base:0.8 ## local variables -ENV ENVIRONMENT docker -ENV PUPPET /opt/puppetlabs/bin/puppet -ENV ROOT_PUPPET /etc/puppetlabs -ENV MODULES $ROOT_PUPPET/code/modules -ENV CONTRIB_MODULES $ROOT_PUPPET/code/modules_contrib +ENV ENVIRONMENT=docker\ + PUPPET=/opt/puppetlabs/bin/puppet\ + ROOT_PUPPET=/etc/puppetlabs +ENV MODULES=$ROOT_PUPPET/code/modules\ + CONTRIB_MODULES=$ROOT_PUPPET/code/modules_contrib ## copy files into container COPY hiera $ROOT_PUPPET/puppet/hiera diff --git a/dockerfile/sklearn.dockerfile b/dockerfile/sklearn.dockerfile index b14a49e04..3640e07a4 100644 --- a/dockerfile/sklearn.dockerfile +++ b/dockerfile/sklearn.dockerfile @@ -1,12 +1,12 @@ -FROM jeff1evesque/ml-base:0.7 +FROM jeff1evesque/ml-base:0.8 ## local variables -ENV ENVIRONMENT docker -ENV PUPPET /opt/puppetlabs/bin/puppet -ENV ROOT_PROJECT /var/machine-learning -ENV ROOT_PUPPET /etc/puppetlabs -ENV MODULES $ROOT_PUPPET/code/modules -ENV CONTRIB_MODULES $ROOT_PUPPET/code/modules_contrib +ENV ENVIRONMENT=docker\ + PUPPET=/opt/puppetlabs/bin/puppet\ + ROOT_PROJECT=/var/machine-learning\ + ROOT_PUPPET=/etc/puppetlabs +ENV MODULES=$ROOT_PUPPET/code/modules\ + CONTRIB_MODULES=$ROOT_PUPPET/code/modules_contrib ## copy files into container COPY hiera $ROOT_PROJECT/hiera diff --git a/dockerfile/webserver.dockerfile b/dockerfile/webserver.dockerfile index 18003fdb4..3f5c23fa3 100644 --- a/dockerfile/webserver.dockerfile +++ b/dockerfile/webserver.dockerfile @@ -1,12 +1,12 @@ -FROM jeff1evesque/ml-sklearn:0.7 +FROM jeff1evesque/ml-sklearn:0.8 ## local variables -ENV ENVIRONMENT docker -ENV PUPPET /opt/puppetlabs/bin/puppet -ENV ROOT_PROJECT /var/machine-learning -ENV ROOT_PUPPET /etc/puppetlabs -ENV MODULES $ROOT_PUPPET/code/modules -ENV CONTRIB_MODULES $ROOT_PUPPET/code/modules_contrib +ENV ENVIRONMENT=docker\ + PUPPET=/opt/puppetlabs/bin/puppet\ + ROOT_PROJECT=/var/machine-learning\ + ROOT_PUPPET=/etc/puppetlabs +ENV MODULES=$ROOT_PUPPET/code/modules\ + CONTRIB_MODULES=$ROOT_PUPPET/code/modules_contrib ## ensure directory RUN mkdir -p $ROOT_PROJECT/interface $ROOT_PROJECT/hiera $ROOT_PUPPET/brain $ROOT_PUPPET/test @@ -24,7 +24,7 @@ COPY puppet/environment/$ENVIRONMENT/modules/webserver $ROOT_PUPPET/code/modules ## ## provision with puppet: either build a web, or api webserver image. ## -## docker build -f webserver.dockerfile -t jeff1evesque/ml-webserver:0.7 . +## docker build -f webserver.dockerfile -t jeff1evesque/ml-webserver:0.8 . ## RUN apt-get update && $PUPPET apply -e "class { webserver: \ run => false, \ @@ -34,8 +34,8 @@ RUN apt-get update && $PUPPET apply -e "class { webserver: \ ## ## executed everytime container starts ## -## docker run --hostname webserver-api --name webserver-api -d jeff1evesque/ml-webserver:0.7 api 0.0.0.0 6001 6 -## docker run --hostname webserver-web --name webserver-web -d jeff1evesque/ml-webserver:0.7 web 0.0.0.0 5001 6 +## docker run --hostname webserver-api --name webserver-api -d jeff1evesque/ml-webserver:0.8 api 0.0.0.0 6001 6 +## docker run --hostname webserver-web --name webserver-web -d jeff1evesque/ml-webserver:0.8 web 0.0.0.0 5001 6 ## docker run\ ## --name webserver\ ## --net=app_nw\ @@ -47,7 +47,7 @@ RUN apt-get update && $PUPPET apply -e "class { webserver: \ ## -v "${PROJECT_ROOT}/hiera:/var/machine-learning/hiera"\ ## -v "${PROJECT_ROOT}/brain:/var/machine-learning/brain"\ ## -v "${PROJECT_ROOT}/test:/var/machine-learning/test"\ -## -d jeff1evesque/ml-webserver:0.7\ +## -d jeff1evesque/ml-webserver:0.8\ ## test | sudo tee pytest.log ## WORKDIR $ROOT_PROJECT diff --git a/hiera/database.yaml b/hiera/database.yaml index b05179c46..a6ec75ad2 100644 --- a/hiera/database.yaml +++ b/hiera/database.yaml @@ -28,8 +28,8 @@ database: username: 'authenticated' password: 'password' keyserver: 'hkp://keyserver.ubuntu.com:80' - mongodb_key: '0C49F3730359A14518585931BC711F9BA15703C6' - source_list: 'deb [ arch=amd64 ] http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.4 multiverse' + mongodb_key: '9DA31620334BD75D9DCB49F368818C72E52529D4' + source_list: 'deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse' storage: dbPath: diff --git a/hiera/packages.yaml b/hiera/packages.yaml index d67a2f9c0..da79dc6ad 100644 --- a/hiera/packages.yaml +++ b/hiera/packages.yaml @@ -9,19 +9,24 @@ system: packages: apt: general: - inotify-tools: '3.14-1ubuntu1' - libssl-dev: '1.0.1f-1ubuntu2.2*' - openssl: '1.0.1f-1ubuntu2.2*' - python-dev: '2.7.5-5ubuntu3' + apt-transport-https: '1.2.*' + inotify-tools: '3.14-1ubuntu*' + libssl-dev: '1.0.2g-1ubuntu*' + openssl: '1.0.2g-1ubuntu*' + python3: '3.5*' + python-dev: '2.7*' + python3-pip: '8.1.1-2ubuntu*' - pip: + pip3: general: - flask-script: '2.0.5' - flask-jwt-extended: '3.3.4' - voluptuous: '0.10.5' + flask-script: '2.0.6' + flask-jwt-extended: '3.13.1' + voluptuous: '0.11.5' pytest-flask: '0.10.0' - six: '1.5.2' - xmltodict: '0.10.1' - scrypt: '0.8.0' - pymongo: '3.4.0' + six: '1.11.0' + xmltodict: '0.11.0' + scrypt: '0.8.6' + pymongo: '3.7.1' mlxtend: '0.13.0' + requests: '2.9.1' + flask: '1.0.2' diff --git a/hiera/reverse-proxy/nginx-api.yaml b/hiera/reverse-proxy/nginx-api.yaml index d31238255..d3ebc0f85 100644 --- a/hiera/reverse-proxy/nginx-api.yaml +++ b/hiera/reverse-proxy/nginx-api.yaml @@ -9,7 +9,6 @@ reverse_proxy: type: 'api' vhost: 'machine-learning-api.com' - host_port: 9090 listen_port: 6000 proxy: 'api-reverse-proxy' members: diff --git a/hiera/reverse-proxy/nginx-web.yaml b/hiera/reverse-proxy/nginx-web.yaml index ccb832da2..c3b21f3d0 100644 --- a/hiera/reverse-proxy/nginx-web.yaml +++ b/hiera/reverse-proxy/nginx-web.yaml @@ -9,7 +9,6 @@ reverse_proxy: type: 'web' vhost: 'machine-learning.com' - host_port: 8080 listen_port: 5000 proxy: 'web-reverse-proxy' members: diff --git a/install_rancher b/install_rancher index ca884fcd5..5ccd42135 100644 --- a/install_rancher +++ b/install_rancher @@ -10,9 +10,9 @@ ## - https://github.com/rancher/rancher/releases/tag/v1.6.18 ## CWD=$(pwd) -RANCHER_CLI_VERSION='v0.6.9' -RANCHER_AGENT_VERSION='v1.2.10' -RANCHER_SERVER_VERSION='v1.6.18' +RANCHER_CLI_VERSION='v0.6.11' +RANCHER_AGENT_VERSION='v1.2.11' +RANCHER_SERVER_VERSION='v1.6.22' RANCHER_REPO='https://github.com/rancher/cli' BOOT2DOCKER_REPO='https://github.com/boot2docker/boot2docker' OPERATING_SYSTEM=$(uname -s) diff --git a/puppet/environment/docker/Puppetfile b/puppet/environment/docker/Puppetfile index 0b394c310..5ce63b27e 100644 --- a/puppet/environment/docker/Puppetfile +++ b/puppet/environment/docker/Puppetfile @@ -4,33 +4,28 @@ mod 'stdlib', :git => 'https://github.com/puppetlabs/puppetlabs-stdlib.git', - :ref => '4.25.1' + :ref => '5.1.0' mod 'apt', :git => 'https://github.com/puppetlabs/puppetlabs-apt.git', - :ref => '4.5.1' + :ref => '6.1.1' mod 'mysql', :git => 'https://github.com/puppetlabs/puppetlabs-mysql.git', - :ref => '3.4.0' + :ref => '6.2.0' mod 'sys', :git => 'https://github.com/counsyl/puppet-sys.git', - :ref => '0.9.18' - -## includes pip, devel, flask, requests, pipx (enhanced pip). -mod 'python', - :git => 'https://github.com/counsyl/puppet-python.git', - :ref => '0.9.9' + :ref => '0.9.23' mod 'timezone', :git => 'https://github.com/BashtonLtd/puppet-timezone.git', - :ref => '2.0.0' + :ref => '2.0.1' mod 'concat', :git => 'https://github.com/puppetlabs/puppetlabs-concat.git', - :ref => '4.2.1' + :ref => '5.1.0' mod 'nginx', :git => 'https://github.com/voxpupuli/puppet-nginx.git', - :ref => 'v0.11.0' \ No newline at end of file + :ref => 'v0.14.0' diff --git a/puppet/environment/docker/modules/mariadb/manifests/client.pp b/puppet/environment/docker/modules/mariadb/manifests/client.pp deleted file mode 100644 index f8a6fd517..000000000 --- a/puppet/environment/docker/modules/mariadb/manifests/client.pp +++ /dev/null @@ -1,8 +0,0 @@ -### -### client.pp, mysql::client: install, and configure mariadb-client. -### -class mariadb::client { - class { '::mysql::client': - package_name => 'mariadb-client', - } -} \ No newline at end of file diff --git a/puppet/environment/docker/modules/mariadb/manifests/dependency.pp b/puppet/environment/docker/modules/mariadb/manifests/dependency.pp index 912ed1d8e..91f8f182c 100644 --- a/puppet/environment/docker/modules/mariadb/manifests/dependency.pp +++ b/puppet/environment/docker/modules/mariadb/manifests/dependency.pp @@ -2,8 +2,7 @@ ### dependency.pp, ensure directories. ### class mariadb::dependency { - ## python dependencies - contain python + contain system::packages ## local variables $root_dir = $::mariadb::root_dir @@ -13,15 +12,15 @@ if ($redis_version and $redis_version != '*') { package { 'pyyaml': ensure => $pyyaml_version, - provider => 'pip', - require => Class['python'], + provider => 'pip3', + require => Class['system::packages'], } } else { package { 'pyyaml': ensure => 'installed', - provider => 'pip', - require => Class['python'], + provider => 'pip3', + require => Class['system::packages'], } } diff --git a/puppet/environment/docker/modules/mariadb/manifests/init.pp b/puppet/environment/docker/modules/mariadb/manifests/init.pp index 23c811945..9d9902097 100644 --- a/puppet/environment/docker/modules/mariadb/manifests/init.pp +++ b/puppet/environment/docker/modules/mariadb/manifests/init.pp @@ -22,7 +22,6 @@ ) inherits ::mariadb::params { class { 'mariadb::dependency': } -> class { 'mariadb::server': } -> - class { 'mariadb::client': } -> class { 'mariadb::bindings': } -> Class['mariadb'] } diff --git a/puppet/environment/docker/modules/mariadb/manifests/server.pp b/puppet/environment/docker/modules/mariadb/manifests/server.pp index f9eb48a6d..d41f619d6 100644 --- a/puppet/environment/docker/modules/mariadb/manifests/server.pp +++ b/puppet/environment/docker/modules/mariadb/manifests/server.pp @@ -22,6 +22,7 @@ ## class { '::mysql::server': package_name => 'mariadb-server', + service_name => 'mysql', root_password => $root_password, users => { "${db_user}@${db_host}" => { diff --git a/puppet/environment/docker/modules/mongodb/manifests/install.pp b/puppet/environment/docker/modules/mongodb/manifests/install.pp index 7f55f5087..ec2cb6d0f 100644 --- a/puppet/environment/docker/modules/mongodb/manifests/install.pp +++ b/puppet/environment/docker/modules/mongodb/manifests/install.pp @@ -7,7 +7,7 @@ $mongodb_key = $::mongodb::mongodb_key $mongodb_source_list = $::mongodb::mongodb_source_list - ## https://docs.mongodb.com/v3.4/tutorial/install-mongodb-on-ubuntu/ + ## https://docs.mongodb.com/v4.0/tutorial/install-mongodb-on-ubuntu/ exec { 'apt-key-puppetlabs': command => "apt-key adv --keyserver ${keyserver} --recv ${mongodb_key}", unless => "apt-key list | grep ${mongodb_key}", @@ -16,7 +16,7 @@ } file { 'mongodb-list-file': - path => '/etc/apt/sources.list.d/mongodb-org-3.4.list', + path => '/etc/apt/sources.list.d/mongodb-org-4.0.list', content => $mongodb_source_list, require => Exec['apt-key-puppetlabs'], notify => Exec['apt-get-update'], @@ -24,7 +24,7 @@ exec { 'apt-get-update': command => 'apt-get update', - path => '/usr/bin', + path => ['/usr/bin', '/bin'], before => Package['mongodb-org-server'], refreshonly => true, } diff --git a/puppet/environment/docker/modules/mongodb/manifests/params.pp b/puppet/environment/docker/modules/mongodb/manifests/params.pp index 0275ccb83..4656380e8 100644 --- a/puppet/environment/docker/modules/mongodb/manifests/params.pp +++ b/puppet/environment/docker/modules/mongodb/manifests/params.pp @@ -40,8 +40,8 @@ $bindIp = '127.0.0.1' $pidfilepath = '/var/run/mongod.pid' $keyserver = 'hkp://keyserver.ubuntu.com:80' - $mongodb_key = '0C49F3730359A14518585931BC711F9BA15703C6' - $mongodb_source_list = 'deb [ arch=amd64 ] http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.4 multiverse' + $mongodb_key = '9DA31620334BD75D9DCB49F368818C72E52529D4' + $mongodb_source_list = 'deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse' $authorization = 'enabled' $hostname = 'mongodb' $username = 'authenticated' diff --git a/puppet/environment/docker/modules/reverse_proxy/manifests/init.pp b/puppet/environment/docker/modules/reverse_proxy/manifests/init.pp index d5b80dd8a..103d8033c 100644 --- a/puppet/environment/docker/modules/reverse_proxy/manifests/init.pp +++ b/puppet/environment/docker/modules/reverse_proxy/manifests/init.pp @@ -10,7 +10,6 @@ $run = $::reverse_proxy::params::run, $type = $::reverse_proxy::params::type, $vhost = $::reverse_proxy::params::vhost, - $host_port = $::reverse_proxy::params::host_port, $listen_port = $::reverse_proxy::params::listen_port, $members = $::reverse_proxy::params::members, $proxy = $::reverse_proxy::params::proxy, diff --git a/puppet/environment/docker/modules/reverse_proxy/manifests/params.pp b/puppet/environment/docker/modules/reverse_proxy/manifests/params.pp index 6ec11318b..049e43448 100644 --- a/puppet/environment/docker/modules/reverse_proxy/manifests/params.pp +++ b/puppet/environment/docker/modules/reverse_proxy/manifests/params.pp @@ -12,7 +12,6 @@ $yaml = true $type = $hiera['type'] $vhost = $hiera['vhost'] - $host_port = $hiera['host_port'] $listen_port = $hiera['listen_port'] $members = $hiera['members'] $proxy = $hiera['proxy'] diff --git a/puppet/environment/docker/modules/sklearn/manifests/install.pp b/puppet/environment/docker/modules/sklearn/manifests/install.pp index a7fcc78d4..0af4d56c8 100644 --- a/puppet/environment/docker/modules/sklearn/manifests/install.pp +++ b/puppet/environment/docker/modules/sklearn/manifests/install.pp @@ -2,7 +2,7 @@ ### install.pp, install sklearn. ### class sklearn::install { - include python + include system::packages ## local variables $root_dir = $::sklearn::root_dir @@ -33,15 +33,15 @@ if ($scikit_learn and $scikit_learn != '*') { package { "scikit_learn==${scikit_learn}": ensure => 'installed', - provider => 'pip', - require => Class['python'], + provider => 'pip3', + require => Class['system::packages'], } } else { package { 'scikit_learn': ensure => 'installed', - provider => 'pip', - require => Class['python'], + provider => 'pip3', + require => Class['system::packages'], } } } \ No newline at end of file diff --git a/puppet/environment/docker/modules/system/manifests/packages.pp b/puppet/environment/docker/modules/system/manifests/packages.pp index 0836594d8..bbb68b265 100644 --- a/puppet/environment/docker/modules/system/manifests/packages.pp +++ b/puppet/environment/docker/modules/system/manifests/packages.pp @@ -3,7 +3,6 @@ ### class system::packages { contain apt - contain python ## local variables $packages = $::system::packages @@ -11,7 +10,7 @@ ## iterate 'packages' hash if $packages { $packages.each |String $provider, $providers| { - if ($provider in ['apt', 'pip']) { + if ($provider in ['apt', 'pip3']) { if has_key($providers, 'general') { $providers['general'].each|String $package, String $version| { package { $package: @@ -19,7 +18,6 @@ provider => $provider, require => [ Class['apt'], - Class['python'], ], } } @@ -27,4 +25,4 @@ } } } -} \ No newline at end of file +} diff --git a/puppet/environment/docker/modules/webserver/manifests/init.pp b/puppet/environment/docker/modules/webserver/manifests/init.pp index a8dbf7790..045dc33f5 100644 --- a/puppet/environment/docker/modules/webserver/manifests/init.pp +++ b/puppet/environment/docker/modules/webserver/manifests/init.pp @@ -8,20 +8,22 @@ ### - https://github.com/counsyl/puppet-python.git ### class webserver ( - $run = $::webserver::params::run, - $gunicorn_bind = $::webserver::params::gunicorn_bind, - $gunicorn_port = $::webserver::params::gunicorn_port, - $gunicorn_workers = $::webserver::params::gunicorn_workers, - $gunicorn_type = $::webserver::params::gunicorn_type, - $gunicorn_version = $::webserver::params::gunicorn_version, - $root_dir = $::webserver::params::root_dir, - $version = $::webserver::params::version, - $flask_log_path = $::webserver::params::flask_log_path, - $pyyaml_version = $::webserver::params::pyyaml_version, - $redis_version = $::webserver::params::redis_version, - $pytest_cov_version = $::webserver::params::pytest_cov_version, - $root_puppet = $::webserver::params::root_puppet, - $platform = $::webserver::params::platform, + $run = $::webserver::params::run, + $gunicorn_bind = $::webserver::params::gunicorn_bind, + $gunicorn_port = $::webserver::params::gunicorn_port, + $gunicorn_workers = $::webserver::params::gunicorn_workers, + $gunicorn_type = $::webserver::params::gunicorn_type, + $gunicorn_version = $::webserver::params::gunicorn_version, + $root_dir = $::webserver::params::root_dir, + $version = $::webserver::params::version, + $flask_log_path = $::webserver::params::flask_log_path, + $pyyaml_version = $::webserver::params::pyyaml_version, + $redis_version = $::webserver::params::redis_version, + $libmysqlclient_version = $::webserver::params::libmysqlclient_version, + $mysqlclient_version = $::webserver::params::mysqlclient_version, + $pytest_cov_version = $::webserver::params::pytest_cov_version, + $root_puppet = $::webserver::params::root_puppet, + $platform = $::webserver::params::platform, ) inherits ::webserver::params { class { 'webserver::install': } -> class { 'webserver::config': } ~> diff --git a/puppet/environment/docker/modules/webserver/manifests/install.pp b/puppet/environment/docker/modules/webserver/manifests/install.pp index 091ce1016..282ebce48 100644 --- a/puppet/environment/docker/modules/webserver/manifests/install.pp +++ b/puppet/environment/docker/modules/webserver/manifests/install.pp @@ -3,44 +3,55 @@ ### class webserver::install { ## python dependencies - include python - include python::flask - include python::requests + include system::packages ## local variables - $pyyaml_version = $::webserver::pyyaml_version - $redis_version = $::webserver::redis_version - $pytest_cov_version = $::webserver::pytest_cov_version - $jest_cli_version = $::webserver::jest_cli_version - $gunicorn_version = $::webserver::version - $platform = $::webserver::platform + $pyyaml_version = $::webserver::pyyaml_version + $redis_version = $::webserver::redis_version + $libmysqlclient_version = $::webserver::libmysqlclient_version + $mysqlclient_version = $::webserver::mysqlclient_version + $pytest_cov_version = $::webserver::pytest_cov_version + $jest_cli_version = $::webserver::jest_cli_version + $gunicorn_version = $::webserver::version + $platform = $::webserver::platform ## development packages if ($platform == 'development') { package { 'pytest-cov': - ensure => $pytest_cov_version, - provider => 'pip', - require => Class['python'], + ensure => $pytest_cov_version, + provider => 'pip3', + require => Class['system::packages'], } } ## pyyaml package { 'pyyaml': - ensure => $pyyaml_version, - provider => 'pip', - require => Class['python'], + ensure => $pyyaml_version, + provider => 'pip3', + require => Class['system::packages'], } ## redis client package { 'redis': - ensure => $redis_version, - provider => 'pip', - require => Class['python'], + ensure => $redis_version, + provider => 'pip3', + require => Class['system::packages'], } - ## mariadb client - class { '::mysql::client': - package_name => 'mariadb-client', + ## + ## mariadb client: workaround for python3x. By default 'MySQLdb' only + ## supported on python2x. + ## + ## - https://raspberrypi.stackexchange.com/a/78217 + ## + package { 'libmysqlclient-dev': + ensure => $libmysqlclient_version, + require => Class['system::packages'], + } + package { 'mysqlclient': + ensure => $mysqlclient_version, + provider => 'pip3', + require => Class['system::packages'], } ## mariadb bindings @@ -50,7 +61,8 @@ ## install gunicorn package { 'gunicorn': - ensure => $gunicorn_version, - provider => 'pip', + ensure => $gunicorn_version, + provider => 'pip3', + require => Class['system::packages'], } } diff --git a/puppet/environment/docker/modules/webserver/manifests/params.pp b/puppet/environment/docker/modules/webserver/manifests/params.pp index e48feb8ce..b46eb41f1 100644 --- a/puppet/environment/docker/modules/webserver/manifests/params.pp +++ b/puppet/environment/docker/modules/webserver/manifests/params.pp @@ -2,31 +2,33 @@ ### params.pp: default class parameters. ### class webserver::params { - $hiera = lookup( { 'name' => 'webserver', 'default_value' => false } ) - $run = true - $pyyaml_version = 'installed' - $redis_version = 'installed' - $pytest_cov_version = 'installed' - $jest_cli_version = 'installed' - $gunicorn_version = 'installed' - $root_puppet = '/etc/puppetlabs' - $platform = 'production' + $hiera = lookup( { 'name' => 'webserver', 'default_value' => false } ) + $run = true + $pyyaml_version = 'installed' + $redis_version = 'installed' + $libmysqlclient_version = 'installed' + $mysqlclient_version = 'installed' + $pytest_cov_version = 'installed' + $jest_cli_version = 'installed' + $gunicorn_version = 'installed' + $root_puppet = '/etc/puppetlabs' + $platform = 'production' if $hiera { - $root_dir = $hiera['root_dir'] - $flask_log_path = $hiera['flask']['log_path'] - $gunicorn_bind = $hiera['gunicorn']['bind'] - $gunicorn_port = $hiera['gunicorn']['port'] - $gunicorn_workers = $hiera['gunicorn']['workers'] - $gunicorn_type = $hiera['gunicorn']['type'] + $root_dir = $hiera['root_dir'] + $flask_log_path = $hiera['flask']['log_path'] + $gunicorn_bind = $hiera['gunicorn']['bind'] + $gunicorn_port = $hiera['gunicorn']['port'] + $gunicorn_workers = $hiera['gunicorn']['workers'] + $gunicorn_type = $hiera['gunicorn']['type'] } else { - $root_dir = '/var/machine-learning' - $flask_log_path = '/var/log/webserver/flask.log' - $gunicorn_bind = '0.0.0.0' - $gunicorn_port = '' - $gunicorn_workers = 6 - $gunicorn_type = 'web' + $root_dir = '/var/machine-learning' + $flask_log_path = '/var/log/webserver/flask.log' + $gunicorn_bind = '0.0.0.0' + $gunicorn_port = '' + $gunicorn_workers = 6 + $gunicorn_type = 'web' } } diff --git a/puppet/environment/docker/modules/webserver/templates/entrypoint.erb b/puppet/environment/docker/modules/webserver/templates/entrypoint.erb index 319245e4f..09c94fd16 100644 --- a/puppet/environment/docker/modules/webserver/templates/entrypoint.erb +++ b/puppet/environment/docker/modules/webserver/templates/entrypoint.erb @@ -14,7 +14,7 @@ CONFDIR=<%= @root_puppet %>/puppet ## run application if [ "$GUNICORN_TYPE" = 'test' ]; then - python app.py test + python3 app.py test else gunicorn \ diff --git a/test/backend b/test/backend index f8178d59c..505518726 100644 --- a/test/backend +++ b/test/backend @@ -25,12 +25,14 @@ fi ## - stop all docker containers ## - remove all docker containers ## - remove all docker images +## - remove all docker dangling images ## - remove all docker volumes ## - delete all docker networks ## if [ "$RECYCLE" = 'true' ] && which docker >/dev/null 2>&1; then sudo docker ps -q | xargs -r docker kill sudo docker ps -a -q | xargs -r docker rm + sudo docker rmi $(docker images -q) sudo docker images -q -f dangling=true | xargs -r docker rmi sudo docker volume ls -qf dangling=true | xargs -r docker volume rm sudo docker network ls -q | xargs -r docker network rm diff --git a/test/live_server/1_authentication/pytest_1_crypto.py b/test/live_server/1_authentication/pytest_1_crypto.py index 8e470befc..44863e74b 100644 --- a/test/live_server/1_authentication/pytest_1_crypto.py +++ b/test/live_server/1_authentication/pytest_1_crypto.py @@ -15,7 +15,7 @@ def test_hashing(): cryptopath = root + '/brain/converter/crypto.py' crypto = imp.load_source('crypto', cryptopath) except Exception as error: - print error + print(error) passwords = ['blue', 'red', 'green', 'yellow'] diff --git a/test/live_server/1_authentication/pytest_3_validate_password.py b/test/live_server/1_authentication/pytest_3_validate_password.py index d1cdfacbb..a4767c9dd 100644 --- a/test/live_server/1_authentication/pytest_3_validate_password.py +++ b/test/live_server/1_authentication/pytest_3_validate_password.py @@ -17,7 +17,7 @@ def test_validate_password(): validate_password = root + '/brain/validator/password.py' vp = imp.load_source('validate_password', validate_password) except yaml.YAMLError as error: - print error + print(error) passwords = [ 'abcdefghij123', # just right diff --git a/test/live_server/3_programmatic_interface/results/pytest_1_svm_prediction.py b/test/live_server/3_programmatic_interface/results/pytest_1_svm_prediction.py index 4cd29507c..af2570632 100644 --- a/test/live_server/3_programmatic_interface/results/pytest_1_svm_prediction.py +++ b/test/live_server/3_programmatic_interface/results/pytest_1_svm_prediction.py @@ -97,13 +97,13 @@ def save_prediction(): assert res.status_code == 200 if res.json['status'] == 1: - print 'Unsuccessful storing the prediction result.' + print('Unsuccessful storing the prediction result.') assert False elif res.json['status'] == 2: - print 'Status was not valid.' + print('Status was not valid.') assert False elif res.json['status'] == 3: - print 'Improper request submitted.' + print('Improper request submitted.') assert False else: assert res.json['status'] == 0 @@ -134,10 +134,10 @@ def retrieve_prediction(): assert res.status_code == 200 if res.json['status'] == 1: - print 'Unsuccessful retrieval of specified prediction parameter.' + print('Unsuccessful retrieval of specified prediction parameter.') assert False elif res.json['status'] == 2: - print 'Improper request submitted.' + print('Improper request submitted.') assert False else: assert res.json['status'] == 0 @@ -188,10 +188,10 @@ def retrieve_prediction_titles(): assert res.status_code == 200 if res.json['status'] == 1: - print 'Unsuccessful retrieval of prediction titles.' + print('Unsuccessful retrieval of prediction titles.') assert False elif res.json['status'] == 2: - print 'Improper request submitted.' + print('Improper request submitted.') assert False else: assert res.json['status'] == 0 diff --git a/test/live_server/3_programmatic_interface/results/pytest_2_svr_prediction.py b/test/live_server/3_programmatic_interface/results/pytest_2_svr_prediction.py index ef21cf669..1499d31c7 100644 --- a/test/live_server/3_programmatic_interface/results/pytest_2_svr_prediction.py +++ b/test/live_server/3_programmatic_interface/results/pytest_2_svr_prediction.py @@ -97,13 +97,13 @@ def save_prediction(): assert res.status_code == 200 if res.json['status'] == 1: - print 'Unsuccessful storing the prediction result.' + print('Unsuccessful storing the prediction result.') assert False elif res.json['status'] == 2: - print 'Status was not valid.' + print('Status was not valid.') assert False elif res.json['status'] == 3: - print 'Improper request submitted.' + print('Improper request submitted.') assert False else: assert res.json['status'] == 0 @@ -136,10 +136,10 @@ def retrieve_prediction(): assert res.status_code == 200 if res.json['status'] == 1: - print 'Unsuccessful retrieval of specified prediction parameter.' + print('Unsuccessful retrieval of specified prediction parameter.') assert False elif res.json['status'] == 2: - print 'Improper request submitted.' + print('Improper request submitted.') assert False else: assert res.json['status'] == 0 @@ -176,10 +176,10 @@ def retrieve_prediction_titles(): assert res.status_code == 200 if res.json['status'] == 1: - print 'Unsuccessful retrieval of prediction titles' + print('Unsuccessful retrieval of prediction titles') assert False elif res.json['status'] == 2: - print 'Improper request submitted.' + print('Improper request submitted.') assert False else: assert res.json['status'] == 0 diff --git a/test/live_server/3_programmatic_interface/results/pytest_3_all_prediction_titles.py b/test/live_server/3_programmatic_interface/results/pytest_3_all_prediction_titles.py index 997a5dbb9..f9bb4c718 100644 --- a/test/live_server/3_programmatic_interface/results/pytest_3_all_prediction_titles.py +++ b/test/live_server/3_programmatic_interface/results/pytest_3_all_prediction_titles.py @@ -96,10 +96,10 @@ def retrieve_prediction_titles(): assert res.status_code == 200 if res.json['status'] == 1: - print 'Unsuccessful retrieval of prediction titles' + print('Unsuccessful retrieval of prediction titles') assert False elif res.json['status'] == 2: - print 'Improper request submitted.' + print('Improper request submitted.') assert False else: assert res.json['status'] == 0