Merge pull request #4 from drucker/feature/pipnize-drucker

Merged!

.gitignore

@@ -1,6 +1,4 @@
 .idea/
 venv/
-drucker_pb2.py
-drucker_pb2_grpc.py
-db.sqlite3
+*.sqlite3
 model/

.gitmodules

@@ -1,8 +0,0 @@
-[submodule "drucker"]
-	path = drucker
-	url = https://github.com/drucker/drucker.git
-	branch = master
-[submodule "drucker-grpc-proto"]
-	path = drucker-grpc-proto
-	url = https://github.com/drucker/drucker-grpc-proto.git
-	branch = master

predict.py → app.py

@@ -1,62 +1,52 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
+
 import traceback
+import csv
+import os
 import io
 
 from enum import Enum
 
-from drucker.logger.logger_jsonlogger import SystemLogger
-from drucker.core.predict_interface import PredictInterface, PredictLabel, PredictResult, EvaluateResult
-from drucker.utils.env_loader import SERVICE_LEVEL_ENUM, APPLICATION_NAME
-from drucker.models import get_model_path, SERVICE_FIRST_BOOT
+from drucker.logger import JsonSystemLogger
+from drucker import Drucker
+from drucker.utils import PredictLabel, PredictResult, EvaluateResult
 
 import numpy as np
-import csv
-import os
 from sklearn.metrics import accuracy_score, precision_recall_fscore_support
 from sklearn.externals import joblib
 
 
-class Predict(PredictInterface):
-    def __init__(self):
-        super().__init__()
-        self.predictor = None
-        self.logger = SystemLogger(logger_name="drucker_predict", app_name=APPLICATION_NAME, app_env=SERVICE_LEVEL_ENUM)
-        self.load_model(get_model_path())
+class MyApp(Drucker):
+    def __init__(self, config_file: str = None):
+        super().__init__(config_file)
+        self.logger = JsonSystemLogger(self.config)
+        self.load_model()
 
     def set_type(self, type_input: Enum, type_output: Enum) -> None:
-        """@deprecated
-        """
         super().set_type(type_input, type_output)
 
     def get_type_input(self) -> Enum:
-        """@deprecated
-        """
         return super().get_type_input()
 
     def get_type_output(self) -> Enum:
-        """@deprecated
-        """
         return super().get_type_output()
 
-    def load_model(self, model_path: str = None) -> None:
+    def load_model(self) -> None:
         """ override
         Load ML model.
-
-        :param model_path:
         :return:
         """
-        assert model_path is not None, \
+        assert self.model_path is not None, \
             'Please specify your ML model path'
         try:
-            self.predictor = joblib.load(model_path)
-
+            self.predictor = joblib.load(self.model_path)
         except Exception as e:
             self.logger.error(str(e))
             self.logger.error(traceback.format_exc())
             self.predictor = None
-            if not SERVICE_FIRST_BOOT:
+            if not self.is_first_boot():
                 # noinspection PyProtectedMember
                 os._exit(-1)
 
@@ -84,7 +74,7 @@ def predict(self, input: PredictLabel, option: dict = None) -> PredictResult:
 
     def evaluate(self, file: bytes) -> EvaluateResult:
         """ override
-        [WIP] Evaluate.
+        [WIP] Evaluate
 
         :param file: Evaluation data file. bytes
         :return:

requirements.txt

@@ -1,3 +1,4 @@
+drucker==0.4.0
 scikit-learn==0.19.1
 numpy==1.14.3
 scipy==1.1.0

sample_model_build.py

@@ -2,16 +2,16 @@
 # -*- coding: utf-8 -*-
 
 import os
-import yaml
 from sklearn.datasets import load_digits
 from sklearn.model_selection import train_test_split
 from sklearn.svm import LinearSVC
 from sklearn.metrics import accuracy_score
 from sklearn.externals import joblib
-from drucker.utils.env_loader import APPLICATION_NAME, DIR_MODEL
+from app import MyApp
 
-config = yaml.load(open("settings.yml", 'r'))
-_DIR_OUTPUT = '{0}/{1}/'.format(DIR_MODEL, APPLICATION_NAME)
+
+app = MyApp("./settings.yml")
+_DIR_OUTPUT = '{0}/{1}/'.format(app.config.DIR_MODEL, app.config.APPLICATION_NAME)
 _FILE_OUTPUT = _DIR_OUTPUT + 'default.model'
 
 

server.py

@@ -6,45 +6,41 @@
 import pathlib
 
 
-root_path = pathlib.Path(os.path.abspath(__file__)).parent
+root_path = pathlib.Path(os.path.abspath(__file__)).parent.parent
 sys.path.append(str(root_path))
-working_path = pathlib.Path(root_path, 'drucker')
-sys.path.append(str(working_path))
 
 
 from concurrent import futures
 import grpc
-import drucker_pb2_grpc
 import time
 
-from drucker.core.drucker_dashboard_servicer import DruckerDashboardServicer
-from drucker.core.drucker_worker_servicer import DruckerWorkerServicer
-from drucker.logger.logger_jsonlogger import SystemLogger, ServiceLogger
-from predict import Predict
-from drucker.utils.env_loader import SERVICE_LEVEL_ENUM, APPLICATION_NAME, SERVICE_PORT
+from drucker import DruckerDashboardServicer, DruckerWorkerServicer
+from drucker.logger import JsonSystemLogger, JsonServiceLogger
+from drucker.protobuf import drucker_pb2_grpc
+from app import MyApp
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 
 def serve():
-    system_logger = SystemLogger(logger_name="drucker", app_name=APPLICATION_NAME, app_env=SERVICE_LEVEL_ENUM)
-    service_logger = ServiceLogger(app_name=APPLICATION_NAME, app_env=SERVICE_LEVEL_ENUM)
-    predictor = Predict()
-    system_logger.info("Wake-up drucker player.")
+    app = MyApp("./settings.yml")
+    system_logger = JsonSystemLogger(app.config)
+    service_logger = JsonServiceLogger(app.config)
+    system_logger.info("Wake-up drucker worker.")
 
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
 
     drucker_pb2_grpc.add_DruckerDashboardServicer_to_server(
-        DruckerDashboardServicer(logger=system_logger, predictor=predictor), server)
+        DruckerDashboardServicer(logger=system_logger, app=app), server)
     drucker_pb2_grpc.add_DruckerWorkerServicer_to_server(
-        DruckerWorkerServicer(logger=service_logger, predictor=predictor), server)
-    server.add_insecure_port("[::]:{0}".format(SERVICE_PORT))
+        DruckerWorkerServicer(logger=service_logger, app=app), server)
+    server.add_insecure_port("[::]:{0}".format(app.config.SERVICE_PORT))
     server.start()
     try:
         while True:
             time.sleep(_ONE_DAY_IN_SECONDS)
     except KeyboardInterrupt:
-        system_logger.info("Shutdown drucker player.")
+        system_logger.info("Shutdown drucker worker.")
         server.stop(0)
 
 

settings.yml

@@ -1,6 +1,9 @@
 # (Mainly) For non-kubernetes users.
 # Define the parameters below.
 
+# Debug flag
+test: True
+
 # This must be unique.
 # You can also determine an environment variable "DRUCKER_APPLICATION_NAME".
 #   Priority is...

start.sh

@@ -7,11 +7,6 @@ set -u
 
 echo "$ECHO_PREFIX Start.."
 
-pip install -r ./drucker-grpc-proto/requirements.txt
-sh ./drucker-grpc-proto/run_codegen.sh
-
-pip install -r ./drucker/requirements.txt
-
 pip install -r requirements.txt
 python sample_model_build.py
 python server.py