Feature/dpnlpf 1924 (#181)

* DPNLPF-1924 add read jinja2 file templates DPNLPF-1924 add tests for UnifiedTemplateMultiLoader action and requirement --------- Co-authored-by: 19950187 <[email protected]>
salute-developers · Mar 19, 2024 · f905d12 · f905d12
1 parent 4563cc7
commit f905d12
Show file tree

Hide file tree

Showing 10 changed files with 190 additions and 17 deletions.
diff --git a/core/basic_models/actions/string_actions.py b/core/basic_models/actions/string_actions.py
@@ -5,14 +5,17 @@
 from functools import cached_property
 from itertools import chain
 from typing import Union, Dict, List, Any, Optional, Tuple, TypeVar, Type
+from lazy import lazy
+from scenarios.user.user_model import User
 
 from core.basic_models.actions.basic_actions import CommandAction
 from core.basic_models.actions.command import Command
 from core.basic_models.answer_items.answer_items import SdkAnswerItem
 from core.model.base_user import BaseUser
 from core.model.factory import list_factory
 from core.text_preprocessing.base import BaseTextPreprocessingResult
-from core.unified_template.unified_template import UnifiedTemplate, UNIFIED_TEMPLATE_TYPE_NAME
+from core.unified_template.unified_template import UnifiedTemplate, UNIFIED_TEMPLATE_TYPE_NAME, \
+    UnifiedTemplateMultiLoader
 
 T = TypeVar("T")
 
@@ -112,6 +115,7 @@ class StringAction(NodeAction):
       }
     }
     """
+
     def __init__(self, items: Dict[str, Any], id: Optional[str] = None):
         super(StringAction, self).__init__(items, id)
 
@@ -130,7 +134,7 @@ def _generate_command_context(self, user: BaseUser, text_preprocessing_result: B
 
     async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult,
                   params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]:
-        # Example: Command("ANSWER_TO_USER", {"answer": {"key1": "string1", "keyN": "stringN"}})
+        # Result command format: Command("ANSWER_TO_USER", {"answer": {"key1": "string1", "keyN": "stringN"}})
         params = params or {}
         command_params = self._generate_command_context(user, text_preprocessing_result, params)
 
@@ -147,6 +151,70 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces
         return commands
 
 
+class StringFileUnifiedTemplateAction(StringAction):
+    @lazy
+    def nodes(self):
+        if self._nodes.get("type", "") == UNIFIED_TEMPLATE_TYPE_NAME:
+            return self._get_template_tree(self._nodes)
+        else:
+            return {k: self._get_template_tree(t) for k, t in self._nodes.items()}
+
+    async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult,
+                  params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]:
+        command_params = dict()
+        params = copy(params) or {}
+        collected = user.parametrizer.collect(text_preprocessing_result, filter_params={"command": self.command})
+        params.update(collected)
+        if type(self.nodes) == UnifiedTemplateMultiLoader:
+            command_params = self._get_rendered_tree(self.nodes, params, self.no_empty_nodes)
+        else:
+            for key, value in self.nodes.items():
+                rendered = self._get_rendered_tree(value, params, self.no_empty_nodes)
+                if rendered != "" or not self.no_empty_nodes:
+                    command_params[key] = rendered
+
+        commands = [Command(self.command, command_params, self.id, request_type=self.request_type,
+                            request_data=self.request_data)]
+        return commands
+
+    def _get_template_tree(self, value):
+        is_dict_unified_template = isinstance(value, dict) and value.get("type") == UNIFIED_TEMPLATE_TYPE_NAME
+        if isinstance(value, str) or is_dict_unified_template:
+            result = UnifiedTemplateMultiLoader(value)
+        elif isinstance(value, dict):
+            result = {}
+            for inner_key, inner_value in value.items():
+                result[inner_key] = self._get_template_tree(inner_value)
+        elif isinstance(value, list):
+            result = []
+            for inner_value in value:
+                result.append(self._get_template_tree(inner_value))
+        else:
+            result = value
+        return result
+
+    def _get_rendered_tree_recursive(self, value, params, no_empty=False):
+        value_type = type(value)
+        if value_type is dict:
+            result = {}
+            for inner_key, inner_value in value.items():
+                rendered = self._get_rendered_tree_recursive(inner_value, params, no_empty=no_empty)
+                if rendered != "" or not no_empty:
+                    result[inner_key] = rendered
+        elif value_type is list:
+            result = []
+            for inner_value in value:
+                rendered = self._get_rendered_tree_recursive(inner_value, params, no_empty=no_empty)
+                if rendered != "" or not no_empty:
+                    result.append(rendered)
+
+        elif value_type is UnifiedTemplateMultiLoader:
+            result = value.render(params)
+        else:
+            result = value
+        return result
+
+
 class AfinaAnswerAction(NodeAction):
     """
     Example:
@@ -161,6 +229,7 @@ class AfinaAnswerAction(NodeAction):
     Output:
     [command1(pronounceText)]
     """
+
     def __init__(self, items: Dict[str, Any], id: Optional[str] = None):
         super(AfinaAnswerAction, self).__init__(items, id)
         self.command: str = ANSWER_TO_USER

diff --git a/core/basic_models/requirement/basic_requirements.py b/core/basic_models/requirement/basic_requirements.py
@@ -16,7 +16,7 @@
 from core.model.registered import Registered
 from core.text_preprocessing.base import BaseTextPreprocessingResult
 from core.text_preprocessing.preprocessing_result import TextPreprocessingResult
-from core.unified_template.unified_template import UnifiedTemplate
+from core.unified_template.unified_template import UnifiedTemplate, UnifiedTemplateMultiLoader
 from core.utils.stats_timer import StatsTimer
 from scenarios.scenario_models.field.field_filler_description import IntersectionFieldFiller
 from scenarios.user.user_model import User
@@ -208,7 +208,7 @@ def _check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: B
 class TemplateRequirement(Requirement):
     def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None:
         super().__init__(items, id)
-        self._template = UnifiedTemplate(items["template"])
+        self._template = UnifiedTemplateMultiLoader(items["template"])
 
     def _check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser,
                params: Dict[str, Any] = None) -> bool:

diff --git a/core/unified_template/unified_template.py b/core/unified_template/unified_template.py
@@ -3,6 +3,7 @@
 from copy import copy
 import jinja2
 from distutils.util import strtobool
+import os
 
 import core.logging.logger_constants as log_const
 from core.logging.logger_utils import log
@@ -84,3 +85,26 @@ def silent_render(self, params_dict):
 
     def __str__(self):
         return str(self.input)
+
+
+class UnifiedTemplateMultiLoader(UnifiedTemplate):
+    """
+    Загружает шаблон jinja из файла или строки
+    Файлы шаблонов jinja должны находиться в директории some-app.static.templates
+    """
+
+    def __init__(self, input):
+        if isinstance(input, dict) and input.get("file"):
+            file_name = input["file"]
+            from smart_kit.configs import get_app_config
+            app_config = get_app_config()
+            templates_dir = os.path.join(app_config.STATIC_PATH, "references/templates")
+            for root, dirs, files in os.walk(templates_dir):
+                if file_name in files:
+                    with open(os.path.join(root, file_name)) as f:
+                        input["template"] = f.read()
+                    super().__init__(input)
+                    return
+            raise Exception(f"Template {file_name} does not exist in templates directory {templates_dir}")
+        else:
+            super().__init__(input)
diff --git a/smart_kit/resources/__init__.py b/smart_kit/resources/__init__.py
@@ -19,7 +19,7 @@
 from core.basic_models.actions.smartpay import SmartPayCreateAction, SmartPayPerformAction, SmartPayGetStatusAction, \
     SmartPayConfirmAction, SmartPayDeleteAction, SmartPayRefundAction
 from core.basic_models.actions.string_actions import StringAction, AfinaAnswerAction, SDKAnswer, \
-    SDKAnswerToUser
+    SDKAnswerToUser, StringFileUnifiedTemplateAction
 from core.basic_models.actions.variable_actions import ClearVariablesAction, DeleteVariableAction, \
     SetLocalVariableAction, SetVariableAction
 from core.basic_models.answer_items.answer_items import items_factory, SdkAnswerItem, answer_items, BubbleText, \
@@ -308,7 +308,7 @@ def init_actions(self):
         actions["self_service_with_state"] = SelfServiceActionWithState
         actions["set_local_variable"] = SetLocalVariableAction
         actions["set_variable"] = SetVariableAction
-        actions["string"] = StringAction
+        actions["string"] = StringFileUnifiedTemplateAction
         actions["push"] = PushAction
         actions["push_authentication"] = PushAuthenticationActionHttp
         actions["push_http"] = PushActionHttp

diff --git a/smart_kit/template/static/references/forms/hello_form.json b/smart_kit/template/static/references/forms/hello_form.json
@@ -61,13 +61,11 @@
             "command": "ANSWER_TO_USER",
             "nodes": {
               "pronounceText": "Сколько лет ты программируешь на Python?",
-              "items": [
-                {
-                  "bubble": {
-                    "text": "Сколько лет ты программируешь на Python?"
-                  }
-                }
-              ]
+              "items": {
+                "type": "unified_template",
+                "file": "experience_items_template.jinja2",
+                "loader": "json"
+              }
             }
           }
         ],

diff --git a/smart_kit/template/static/references/templates/experience_items_template.jinja2 b/smart_kit/template/static/references/templates/experience_items_template.jinja2
@@ -0,0 +1,9 @@
+{{
+    [
+        {
+            "bubble": {
+                "text": "Сколько лет ты программируешь на Python?"
+            }
+        }
+    ] |tojson
+}}
diff --git a/tests/pure_framework/core_tests/basic_scenario_models_test/action_test/test_action.py b/tests/pure_framework/core_tests/basic_scenario_models_test/action_test/test_action.py
@@ -1,5 +1,6 @@
 # coding: utf-8
 import json
+import os
 import unittest
 import uuid
 from unittest.mock import Mock, MagicMock, patch, AsyncMock
@@ -29,15 +30,16 @@
 from core.basic_models.actions.external_actions import ExternalAction
 from core.basic_models.actions.push_action import PushAction, PushAuthenticationActionHttp, PushActionHttp, \
     GetRuntimePermissionsAction
-from core.basic_models.actions.string_actions import StringAction, AfinaAnswerAction, SDKAnswer, \
-    SDKAnswerToUser, NodeAction
+from core.basic_models.actions.string_actions import StringAction, StringFileUnifiedTemplateAction, \
+    AfinaAnswerAction, SDKAnswer, SDKAnswerToUser, NodeAction
 from core.basic_models.answer_items.answer_items import SdkAnswerItem, items_factory, answer_items, BubbleText, \
     ItemCard, PronounceText, SuggestText, SuggestDeepLink
 from core.basic_models.requirement.basic_requirements import requirement_factory, Requirement, requirements
 from core.model.registered import registered_factories
 from core.text_preprocessing.base import BaseTextPreprocessingResult
 from core.unified_template.unified_template import UnifiedTemplate, UNIFIED_TEMPLATE_TYPE_NAME
 from smart_kit.action.http import HTTPRequestAction
+from smart_kit import configs
 from smart_kit.request.kafka_request import SmartKitKafkaRequest
 from smart_kit.utils.picklable_mock import PicklableMock, PicklableMagicMock
 
@@ -187,6 +189,44 @@ async def test_string_action(self):
         self.assertEqual(expected[0].name, result[0].name)
         self.assertEqual(expected[0].payload, result[0].payload)
 
+    async def test_string_file_unified_template_action_inline(self):
+        expected = [Command("cmd_id", {"item": "template", "params": "params"})]
+        user = PicklableMagicMock()
+        template = PicklableMock()
+        template.get_template = Mock(return_value=["nlpp.payload.personInfo.identityCard"])
+        user.descriptions = {"render_templates": template}
+        params = {"params": "params"}
+        user.parametrizer = MockSimpleParametrizer(user, {"data": params})
+        items = {"command": "cmd_id", "nodes": {"item": "template", "params": "{{params}}"}}
+        action = StringFileUnifiedTemplateAction(items)
+        result = await action.run(user, None)
+        self.assertEqual(expected[0].name, result[0].name)
+        self.assertEqual(expected[0].payload, result[0].payload)
+
+    async def test_string_file_unified_template_action_from_file(self):
+        expected = [Command("cmd_id", {"item": "template", "params": "params"})]
+        user = PicklableMagicMock()
+        template = PicklableMock()
+        template.get_template = Mock(return_value=["nlpp.payload.personInfo.identityCard"])
+        user.descriptions = {"render_templates": template}
+        params = {"params": "params"}
+        user.parametrizer = MockSimpleParametrizer(user, {"data": params})
+        items = {
+            "command": "cmd_id",
+            "nodes": {
+                "type": "unified_template",
+                "file": "test_string_file_unified_template_action.jinja2",
+                "loader": "json"
+            }
+        }
+        static_path = os.path.join(os.path.dirname(configs.__file__), os.pardir, os.pardir, "tests", "static")
+        cfg = MagicMock(STATIC_PATH=static_path)
+        configs.get_app_config = PicklableMagicMock(return_value=cfg)
+        action = StringFileUnifiedTemplateAction(items)
+        result = await action.run(user, None)
+        self.assertEqual(expected[0].name, result[0].name)
+        self.assertEqual(expected[0].payload, result[0].payload)
+
     async def test_else_action_if(self):
         registered_factories[Requirement] = requirement_factory
         requirements["test"] = MockRequirement
@@ -565,7 +605,8 @@ async def test_push_authentication_action_http_call(self, request_mock: Mock):
                 'headers':
                     {
                         'RqUID': '3f68e69e-351b-4e6f-b251-480f0cb08a5d',
-                        'Authorization': 'Basic QCE4OUZCLjRENjIuM0E1MS5BOUVCITAwMDEhOTZFNS5BRTg5ITAwMDghQjFBRi5EQjdELjE1ODYuODRGMzpzZWNyZXQ='  # noqa
+                        'Authorization': 'Basic QCE4OUZCLjRENjIuM0E1MS5BOUVCITAwMDEhOTZFNS5'
+                                         'BRTg5ITAwMDghQjFBRi5EQjdELjE1ODYuODRGMzpzZWNyZXQ='
                     }
             }
         }

diff --git a/tests/pure_framework/core_tests/requirements_test/test_requirements.py b/tests/pure_framework/core_tests/requirements_test/test_requirements.py
@@ -2,7 +2,7 @@
 import os
 import unittest
 from time import time
-from unittest.mock import Mock, patch
+from unittest.mock import Mock, patch, MagicMock
 
 import smart_kit
 from core.basic_models.classifiers.basic_classifiers import ExternalClassifier
@@ -18,6 +18,7 @@
     NormalizedTextInSetRequirement
 from core.basic_models.variables.variables import Variables
 from core.model.registered import registered_factories
+from smart_kit import configs
 from smart_kit.text_preprocessing.local_text_normalizer import LocalTextNormalizer
 from smart_kit.utils.picklable_mock import PicklableMock
 
@@ -244,6 +245,30 @@ def test_template_req_raise(self):
         with self.assertRaises(TypeError):
             _ = requirement.check(None, user)
 
+    def test_template_req_from_file(self):
+        items = {
+            "template": {
+                "type": "unified_template",
+                "file": "test_unified_template_requirement.jinja2",
+                "loader": "json"
+            }
+        }
+        static_path = os.path.join(os.path.dirname(configs.__file__), os.pardir, os.pardir, "tests", "static")
+        cfg = MagicMock(STATIC_PATH=static_path)
+        configs.get_app_config = MagicMock(return_value=cfg)
+        requirement = TemplateRequirement(items)
+        params = {
+            "payload": {
+                "groupCode": "BROKER",
+                "murexIds": ["AAA", "BBB"],
+                "message": " BBB    "
+            }
+        }
+        user = PicklableMock()
+        user.parametrizer = PicklableMock()
+        user.parametrizer.collect = Mock(return_value=params)
+        self.assertTrue(requirement.check(None, user))
+
     def test_rolling_requirement_true(self):
         user = PicklableMock()
         user.id = "353454"

diff --git a/tests/static/references/templates/test_string_file_unified_template_action.jinja2 b/tests/static/references/templates/test_string_file_unified_template_action.jinja2
@@ -0,0 +1,4 @@
+{
+    "item": "template",
+    "params": "{{params}}"
+}
diff --git a/tests/static/references/templates/test_unified_template_requirement.jinja2 b/tests/static/references/templates/test_unified_template_requirement.jinja2
@@ -0,0 +1,3 @@
+"{{
+    payload.message.strip() in payload.murexIds
+}}"