erge branch '341_fastapi_transition' of https://github.com/nsacyber/W…

…ALKOFF into 341_fastapi_transition

.travis.yml

@@ -4,8 +4,7 @@ os: linux
 language: python
 python:
   - "3.7"
-  #- "3.5-dev" # 3.5 development branch
-  #- "nightly" # points to the latest development branch e.g. 3.6-dev
+  - "3.8"
 
 services:
   - docker
@@ -16,7 +15,6 @@ env:
 before_install:
   - sudo apt-get update
   - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-  - docker version
   - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-compose
   - pip install -U pip
   - pip install -U pytest
@@ -30,13 +28,13 @@ install:
 
 before_script:
   - docker swarm init
-  - ./walkoff.sh up -bdy
+  - ./walkoff.sh up -bdyr
 
 # command to run tests
 script:
-  - coverage run -m pytest testing/api/
+  - coverage run -m pytest testing/api/ --omit ~/virtualenv/*
   #- coverage combine
   - coverage report
 
 after_script:
-  - ./walkoff.sh down -cy
+  - ./walkoff.sh down -cdy

api/server/app.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 from http import HTTPStatus
 import os
@@ -16,7 +17,9 @@
 from api.server.db import mongo, get_mongo_c
 from api.server.scheduler import Scheduler, get_scheduler
 from api.server.utils.problems import ProblemException
-from api.server.security import get_raw_jwt, verify_token_in_decoded, verify_token_not_blacklisted, user_has_correct_roles, \
+from api.server.utils.socketio import sio
+from api.server.security import get_raw_jwt, verify_token_in_decoded, verify_token_not_blacklisted, \
+    user_has_correct_roles, \
     get_roles_by_resource_permission
 
 from common.config import static, config
@@ -28,7 +31,6 @@
 _scheduler = Scheduler()
 p = Path('./apps').glob('**/*')
 
-
 _app.mount("/walkoff/api", _walkoff)
 _app.mount("/walkoff/client", StaticFiles(directory=static.CLIENT_PATH), name="static")
 
@@ -60,6 +62,14 @@ async def start_banner():
     logger.info("API Server started.")
 
 
+@_app.on_event("startup")
+async def connect_to_socketio():
+    logger.info("Connecting to Socket.IO server.")
+    await sio.connect(config.SOCKETIO_URI, socketio_path=static.SOCKETIO_PATH, namespaces=[static.SIO_NS_NODE,
+                                                                                           static.SIO_NS_WORKFLOW,
+                                                                                           static.SIO_NS_BUILD])
+
+
 @_app.on_event("startup")
 async def push_to_minio():
     minio_client = Minio(config.MINIO, access_key=config.get_from_file(config.MINIO_ACCESS_KEY_PATH),
@@ -78,13 +88,26 @@ async def push_to_minio():
 
     files_to_upload = [x for x in p if x.is_file()]
     for file in files_to_upload:
-        path_to_file = str(file)
+        path_to_file = str(file).replace("\\", "/")
         with open(path_to_file, "rb") as file_data:
             file_stat = os.stat(path_to_file)
             minio_client.put_object("apps-bucket", path_to_file, file_data, file_stat.st_size)
 
     logger.info("Apps Pushed to Minio")
 
+
+@_app.on_event("startup")
+async def workflow_results_listener():
+    asyncio.create_task(results.update_workflow_status())
+
+
+@_app.on_event("shutdown")
+async def close_connections():
+    await sio.disconnect()
+    mongo.reg_client.disconnect()
+    await mongo.async_client.disconnect()
+
+
 # Note: The request goes through middleware here in opposite order of instantiation, last to first.
 @_walkoff.middleware("http")
 async def db_session_middleware(request: Request, call_next):
@@ -193,7 +216,8 @@ async def jwt_required_middleware(request: Request, call_next):
                 return e.as_response()
 
             await verify_token_in_decoded(decoded_token=decoded_token, request_type='access')
-            await verify_token_not_blacklisted(walkoff_db=walkoff_db, decoded_token=decoded_token, request_type='access')
+            await verify_token_not_blacklisted(walkoff_db=walkoff_db, decoded_token=decoded_token,
+                                               request_type='access')
 
     response = await call_next(request)
     return response
@@ -208,6 +232,7 @@ async def problem_exception_handler(request: Request, exc: ProblemException):
     )
     return r
 
+
 # Include routers here
 _walkoff.include_router(auth.router,
                         prefix="/auth",
@@ -324,7 +349,6 @@ def custom_openapi():
 
 app = _app
 
-
 # @_app.after_request
 # def after_request(response):
 #     response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, public, max-age=0"

api/server/db/__init__.py

@@ -1,5 +1,7 @@
-from starlette.requests import Request
+from uuid import uuid4
 
+from starlette.requests import Request
+from pydantic import BaseModel
 import motor.motor_asyncio
 import pymongo
 
@@ -9,6 +11,17 @@
 from common.helpers import preset_uuid
 
 
+class IDBaseModel(BaseModel):
+
+    _id_field: str = "id_"
+    _name_field: str = "name"
+
+    def __init__(self, **kwargs):
+        super(IDBaseModel, self).__init__(**kwargs)
+        if not getattr(self, self._id_field):
+            setattr(self, self._id_field, uuid4())
+
+
 class MongoManager(object):
     def __init__(self):
         self.async_client = motor.motor_asyncio.AsyncIOMotorClient(username=config.DB_USERNAME,
@@ -26,6 +39,7 @@ def init_db(self):
         id_index = pymongo.IndexModel([("id_", pymongo.ASCENDING)], unique=True)
         name_index = pymongo.IndexModel([("name", pymongo.ASCENDING)], unique=True)
         username_index = pymongo.IndexModel([("username", pymongo.ASCENDING)], unique=True)
+        execution_index = pymongo.IndexModel([("execution_id", pymongo.ASCENDING)], unique=True)
 
         self.reg_client.walkoff_db.apps.create_indexes([id_index, name_index])
 
@@ -41,6 +55,8 @@ def init_db(self):
 
         self.reg_client.walkoff_db.scheduler.create_indexes([id_index, name_index])
 
+        self.reg_client.walkoff_db.workflowqueue.create_indexes([execution_index])
+
         if "settings" not in self.reg_client.walkoff_db.list_collection_names():
             self.reg_client.walkoff_db.settings.insert_one({
                 "id_": preset_uuid("settings"),

api/server/db/action.py

@@ -4,14 +4,14 @@
 
 from pydantic import BaseModel, ValidationError, validator
 
-# from api.server.db import Base, BaseSchema
+from api.server.db import IDBaseModel
 from api.server.db.parameter import ParameterApiModel, ParameterModel  # ParameterApiSchema, Parameter, ParameterSchema,
 from api.server.db.returns import ReturnApiModel  # ReturnApiSchema,
 
 logger = logging.getLogger(__name__)
 
 
-class ActionApiModel(BaseModel):
+class ActionApiModel(IDBaseModel):
     id_: UUID = None
     name: str
     node_type: str = ""
@@ -21,7 +21,7 @@ class ActionApiModel(BaseModel):
     deprecated: bool = False
 
 
-class ActionModel(BaseModel):
+class ActionModel(IDBaseModel):
     id_: UUID = None
     errors: List[str] = []
     is_valid: bool = True

api/server/db/appapi.py

@@ -5,7 +5,7 @@
 import semver
 from pydantic import BaseModel, ValidationError, validator, UrlStr, EmailStr
 
-# from api.server.db import Base, BaseSchema
+from api.server.db import IDBaseModel
 from api.server.db.action import ActionApiModel #ActionApiSchema,
 
 logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ class AppExternalDocModel(BaseModel):
     url: UrlStr = None
 
 
-class AppApiModel(BaseModel):
+class AppApiModel(IDBaseModel):
     id_: UUID = None
     name: str
     app_version: str = "1.0.0"
@@ -37,7 +37,6 @@ class AppApiModel(BaseModel):
     license_info: AppApiLicenseModel = None
     external_docs: AppExternalDocModel = None
     actions: List[ActionApiModel]
-    _secondary_id = "name"
 
     @validator('app_version')
     def app_version_is_semver(cls, value, values):

api/server/db/branch.py

@@ -3,11 +3,13 @@
 
 from pydantic import BaseModel
 
+from api.server.db import IDBaseModel
+
 
 logger = logging.getLogger(__name__)
 
 
-class BranchModel(BaseModel):
+class BranchModel(IDBaseModel):
     id_: UUID = None
     source_id: UUID
     destination_id: UUID

api/server/db/condition.py

@@ -4,10 +4,12 @@
 
 from pydantic import BaseModel, ValidationError, validator
 
+from api.server.db import IDBaseModel
+
 logger = logging.getLogger(__name__)
 
 
-class ConditionModel(BaseModel):
+class ConditionModel(IDBaseModel):
     id_: UUID = None
     errors: List[str] = []
     is_valid: bool = True

api/server/db/dashboard.py

@@ -6,12 +6,13 @@
 from pydantic import BaseModel
 
 from common.helpers import validate_uuid
+from api.server.db import IDBaseModel
 
 
 logger = logging.getLogger(__name__)
 
 
-class WidgetModel(BaseModel):
+class WidgetModel(IDBaseModel):
     id_: UUID = None
     name: str
     type_: str
@@ -22,11 +23,10 @@ class WidgetModel(BaseModel):
     options: dict = {}
 
 
-class DashboardModel(BaseModel):
+class DashboardModel(IDBaseModel):
     id_: UUID = None
     name: str
     widgets: List[WidgetModel]
-    _secondary_id = "name"
 
 # class Dashboard(Base):
 #     __tablename__ = 'dashboard'

api/server/db/global_variable.py

@@ -3,21 +3,21 @@
 
 from pydantic import BaseModel, Schema
 
+from api.server.db import IDBaseModel
 from api.server.db.permissions import PermissionsModel
 from api.server.utils.helpers import JSON
 
 
 logger = logging.getLogger(__name__)
 
 
-class GlobalVariable(BaseModel):
+class GlobalVariable(IDBaseModel):
     id_: UUID = None
     walkoff_type_: str = "variable"
     name: str
     permissions: PermissionsModel
     value: str
     description: str = None
-    _secondary_id = "name"
 
     class Config:
         schema_extra = {
@@ -46,11 +46,10 @@ class Config:
         }
 
 
-class GlobalVariableTemplate(BaseModel):
+class GlobalVariableTemplate(IDBaseModel):
     id_: UUID = None
     walkoff_type_: str = "variable"
     name: str
     json_schema: JSON = {}
     description: str = None
-    _secondary_id = "name"
 

api/server/db/parameter.py

@@ -6,7 +6,7 @@
 from pydantic import BaseModel, ValidationError, validator
 from common.workflow_types import ParameterVariant
 
-from api.server.db import mongo
+from api.server.db import mongo, IDBaseModel
 # from jsonschema import Draft4Validator, SchemaError, ValidationError as JSONSchemaValidationError
 #
 # from sqlalchemy import Column, ForeignKey, String, JSON, Enum, Boolean
@@ -24,7 +24,7 @@
 logger = logging.getLogger(__name__)
 
 
-class ParameterApiModel(BaseModel):
+class ParameterApiModel(IDBaseModel):
     id_: UUID = None
     name: str
     location: str = ""
@@ -35,7 +35,7 @@ class ParameterApiModel(BaseModel):
     json_schema: dict = {}
 
 
-class ParameterModel(BaseModel):
+class ParameterModel(IDBaseModel):
     id_: UUID = None
     name: str
     variant: ParameterVariant

api/server/db/returns.py

@@ -11,11 +11,12 @@
 # from marshmallow_sqlalchemy import field_for, ModelSchema
 #
 # from api.server.db import Base, BaseSchema
+from api.server.db import IDBaseModel
 
 logger = logging.getLogger(__name__)
 
 
-class ReturnApiModel(BaseModel):
+class ReturnApiModel(IDBaseModel):
     id_: UUID = None
     location: str = ""
     description: str = ""

api/server/db/role.py

@@ -3,15 +3,15 @@
 from pydantic import BaseModel, validator
 from typing import List
 
+from api.server.db import IDBaseModel
 from api.server.db.resource import ResourceModel
 
 
-class RoleModel(BaseModel):
+class RoleModel(IDBaseModel):
     id_: UUID = None
     name: str = None
     description: str = None
     resources: List[ResourceModel] = None
-    _secondary_id = "name"
 
     # def set_resources(self, new_resources, role_col: AsyncIOMotorCollection):
     #     """Adds the given list of resources to the Role object.

api/server/db/scheduledtasks.py

@@ -6,6 +6,7 @@
 from typing import List, Union
 
 from pydantic import BaseModel
+from api.server.db import IDBaseModel
 
 logger = logging.getLogger(__name__)
 
@@ -61,7 +62,7 @@ class IntervalTrigger(BaseModel):
     jitter: int = None
 
 
-class ScheduledTask(BaseModel):
+class ScheduledTask(IDBaseModel):
     id_: UUID = None
     name: str
     description: str = ""