Editor: Fix rendering of html in all page (SEA-1330)

.pre-commit-config.yaml

@@ -27,7 +27,6 @@ repos:
         additional_dependencies:
           - flake8-bugbear==24.2.6
           - flake8-comprehensions==3.14.0
-          - 'flake8-markupsafe@git+https://github.com/vmagamedov/flake8-markupsafe@b391bd13df9330e01666d304b7f4403d67e5ceba'
           - flake8-noqa==1.4.0
           - flake8-pyi==24.1.0
           - flake8-type-checking==2.9.0

setup.cfg

@@ -26,9 +26,11 @@ install_requires =
     arrow
     bcrypt
     Babel
+    bleach
     click
     email_validator
     humanize
+    html2text
     fanstatic
     fasteners
     Markdown

src/privatim/__init__.py

@@ -1,6 +1,8 @@
 from functools import partial
 from fanstatic import Fanstatic
 
+from pyramid.events import BeforeRender
+from privatim import helpers
 from privatim.layouts.action_menu import ActionMenuEntry
 from pyramid.config import Configurator
 from pyramid_beaker import session_factory_from_settings
@@ -107,6 +109,16 @@ def add_action_menu_entries(
         'add_action_menu_entries', reify=True)
 
 
+def add_renderer_globals(event: BeforeRender) -> None:
+    """ Makes the helpers module available in all templates.
+    For example, you can access Markup via 'h':
+
+    <h4 tal:content="h.Markup(activity.title[:100])>
+
+    """
+    event['h'] = helpers
+
+
 def main(
     global_config: Any, **settings: Any
 ) -> 'WSGIApplication':  # pragma: no cover
@@ -130,6 +142,7 @@ def main(
 
     with Configurator(settings=settings, root_factory=root_factory) as config:
         includeme(config)
+        config.add_subscriber(add_renderer_globals, BeforeRender)
 
     app = config.make_wsgi_app()
     return Fanstatic(app, versioning=True)
@@ -164,9 +177,71 @@ def upgrade(context: 'UpgradeContext'):  # type: ignore[no-untyped-def]
             server_default=func.now()
         )
     )
-    context.operations.alter_column(
+
+    context.alter_column(
         'comments',
         'modified',
         new_column_name='updated'
     )
+
+    # drop unused Statements column
+    context.drop_column('user', 'statements')
+    context.drop_table('statements')
+
+    # for column in [
+    #     'title',
+    #     'description',
+    #     'recommendation',
+    #     'evaluation_result',
+    #     'decision',
+    # ]:
+    #     if context.has_column('consultations', column):
+    #         col_info = context.get_column_info('consultations', column)
+    #         if col_info and not isinstance(col_info['type'], MarkupText):
+    #             context.operations.alter_column(
+    #                 'consultations',
+    #                 column,
+    #                 type_=MarkupText,
+    #                 existing_type=col_info['type'],
+    #                 nullable=col_info['nullable']
+    #             )
+    #
+    # # Upgrade Meeting model
+    # for column in ['name', 'decisions']:
+    #     if context.has_column('meetings', column):
+    #         col_info = context.get_column_info('meetings', column)
+    #         if col_info and not isinstance(col_info['type'], MarkupText):
+    #             context.operations.alter_column(
+    #                 'meetings',
+    #                 column,
+    #                 type_=MarkupText,
+    #                 existing_type=col_info['type'],
+    #                 nullable=col_info['nullable']
+    #             )
+    # # Upgrade AgendaItem model
+    # for column in ['title', 'description']:
+    #     if context.has_column('agenda_items', column):
+    #         col_info = context.get_column_info('agenda_items', column)
+    #         if col_info and not isinstance(col_info['type'], MarkupText):
+    #             context.operations.alter_column(
+    #                 'agenda_items',
+    #                 column,
+    #                 type_=MarkupText,
+    #                 existing_type=col_info['type'],
+    #                 nullable=col_info['nullable']
+    #             )
+    #
+    # # Upgrade Comment model
+    # if context.has_table('comments'):
+    #     if context.has_column('comments', 'content'):
+    #         col_info = context.get_column_info('comments', 'content')
+    #         if col_info and not isinstance(col_info['type'], MarkupText):
+    #             context.operations.alter_column(
+    #                 'comments',
+    #                 'content',
+    #                 type_=MarkupText,
+    #                 existing_type=col_info['type'],
+    #                 nullable=col_info['nullable']
+    #             )
+    #
     context.commit()

src/privatim/cli/upgrade.py

@@ -1,8 +1,6 @@
 import logging
 import os
 from enum import Enum
-from typing import TYPE_CHECKING
-from typing import Any
 
 import click
 import plaster
@@ -17,10 +15,12 @@
 from privatim.orm import Base
 
 
+from typing import TYPE_CHECKING, Any
 if TYPE_CHECKING:
     from sqlalchemy import Column as _Column
     from sqlalchemy import Engine
     from sqlalchemy.orm import Session
+    from sqlalchemy.engine.interfaces import ReflectedColumn
 
     Column = _Column[Any]
 
@@ -31,12 +31,12 @@ class UpgradeContext:
 
     def __init__(self, db: 'Session'):
         self.session = db
-        self.engine: Engine = self.session.bind  # type: ignore
+        self.engine: 'Engine' = self.session.bind  # type: ignore
 
         self.operations_connection = db._connection_for_bind(
             self.engine
         )
-        self.operations: Any = Operations(
+        self.operations: Operations = Operations(
             MigrationContext.configure(
                 self.operations_connection
             )
@@ -63,6 +63,18 @@ def add_column(self, table:  str, column: 'Column') -> bool:
                 return True
         return False
 
+    def alter_column(
+        self, table_name: str, column: str, new_column_name: str
+    ) -> bool:
+        if self.has_table(table_name):
+            if (not self.has_column(table_name, new_column_name) and
+                    self.has_column(table_name, column)):
+                self.operations.alter_column(
+                    table_name, column, new_column_name=new_column_name
+                )
+                return True
+        return False
+
     def drop_column(self, table: str, name: str) -> bool:
         if self.has_table(table):
             if self.has_column(table, name):
@@ -89,6 +101,24 @@ def create_foreign_key(self, source_table: str, referent_table: str,
             return True
         return False
 
+    def get_column_info(
+            self,
+            table: str, column: str
+    ) -> 'ReflectedColumn | None':
+        """ Get type information about column. Use like this:
+
+             col_info = context.get_column_info('consultations', column)
+             if col_info and not isinstance(col_info['type'], MarkupText):
+                do_something()
+        """
+
+        inspector = inspect(self.operations_connection)
+        columns = inspector.get_columns(table)
+        for col in columns:
+            if col['name'] == column:
+                return col
+        return None
+
     def get_enum_values(self, enum_name: str) -> set[str]:
         if self.engine.name != 'postgresql':
             return set()

src/privatim/forms/consultation_form.py

@@ -51,10 +51,6 @@ def __init__(
         ]
         self.status.choices = translated_choices
 
-        # If editing, populate the secondary_tags field
-        # if context and context.secondary_tags:
-        #     self.secondary_tags.process_data(context.secondary_tags)
-
     title = TextAreaField(
         _('Title'),
         validators=[DataRequired()],

src/privatim/forms/core.py

@@ -1,8 +1,10 @@
 from functools import partial
 from wtforms import Form as BaseForm
 from wtforms import Label
+from wtforms.fields.simple import TextAreaField
 from wtforms.meta import DefaultMeta
 
+from privatim.html import sanitize_html
 from privatim.i18n import pluralize
 from privatim.i18n import translate
 
@@ -15,8 +17,8 @@
 if TYPE_CHECKING:
     from _typeshed import SupportsItems
     from collections.abc import Callable, Mapping, MutableMapping, Sequence
-    from markupsafe import Markup
     from wtforms import Field
+    from markupsafe import Markup
     from wtforms.fields.core import UnboundField
     from wtforms.form import BaseForm as _BaseForm
     from wtforms.meta import _MultiDictLike
@@ -48,6 +50,16 @@ def ngettext(self, singular: str, plural: str, n: int) -> str:
         return pluralize(singular, plural, n)
 
 
+class HtmlField(TextAreaField):
+    """ A textfield with html with integrated sanitation. """
+
+    data: 'Markup | None'
+
+    def pre_validate(self, form: '_BaseForm') -> None:
+
+        self.data = sanitize_html(self.data)
+
+
 class BootstrapMeta(DefaultMeta):
 
     def bind_field(
@@ -56,6 +68,11 @@ def bind_field(
             unbound_field: 'UnboundField[_FieldT]',
             options:       'MutableMapping[str, Any]'
     ) -> 'Field':
+
+        # If the field is a TextAreaField, replace it with our patched version
+        if unbound_field.field_class is TextAreaField:
+            unbound_field.field_class = HtmlField
+
         # NOTE: This adds bootstrap specific field classes to render_kw
         render_kw = unbound_field.kwargs.get('render_kw', {})
         field_type = unbound_field.field_class.__name__
@@ -80,6 +97,7 @@ def bind_field(
         )
         if not isinstance(field, TransparentFormField):
             field.label = BootstrapLabel(field.label, field.description)
+
         return field
 
     def render_field(

src/privatim/forms/fields/fields.py

@@ -31,8 +31,8 @@
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
     from wtforms.fields.choices import SelectFieldBase
-    from markupsafe import Markup
     from collections.abc import Sequence
+    from markupsafe import Markup
     from datetime import datetime
     from privatim.types import FileDict as StrictFileDict
     from privatim.forms.types import (

src/privatim/helpers.py

@@ -0,0 +1,9 @@
+# Anything imported here will be available globally in templates
+# You can add more imports and functions to helpers.py as necessary to
+# make features available in your templates.
+
+# More info
+# https://docs.pylonsproject.org/projects/pyramid_cookbook/en/latest
+# /templates/templates.html
+
+from markupsafe import Markup   # noqa: F401

src/privatim/html.py

@@ -0,0 +1,69 @@
+
+from bleach.sanitizer import Cleaner  # type:ignore[import-untyped]
+from markupsafe import Markup
+
+
+from typing import TypeVar
+
+
+_StrT = TypeVar('_StrT', bound=str)
+
+
+# html tags allowed by bleach
+SANE_HTML_TAGS = [
+    'a',
+    'abbr',
+    'b',
+    'br',
+    'blockquote',
+    'code',
+    'del',
+    'div',
+    'em',
+    'i',
+    'img',
+    'hr',
+    'li',
+    'ol',
+    'p',
+    'pre',
+    'strong',
+    'sup',
+    'span',
+    'ul',
+    'h1',
+    'h2',
+    'h3',
+    'h4',
+    'h5',
+    'h6',
+    'table',
+    'tbody',
+    'thead',
+    'tr',
+    'th',
+    'td',
+]
+
+# html attributes allowed by bleach
+SANE_HTML_ATTRS = {
+    'a': ['href', 'title'],
+    'abbr': ['title', ],
+    'acronym': ['title', ],
+    'img': ['src', 'alt', 'title']
+}
+
+
+cleaner = Cleaner(
+    tags=SANE_HTML_TAGS,
+    attributes=SANE_HTML_ATTRS
+)
+
+
+def sanitize_html(html: str | None) -> Markup:
+    """ Takes the given html and strips all but a whitelisted number of tags
+    from it.
+
+    """
+
+    return Markup(cleaner.clean(html or ''))

src/privatim/layouts/macros.pt

@@ -94,7 +94,7 @@
             </div>
             <div class="row comment-answer-form-container">
                 <metal:block use-macro="layout.macros['answer_comment_form']"
-                         tal:define="add_comment_route_name request.route_url('add_comment', id=consultation.id,
+                         tal:define="add_comment_route_name request.route_url('add_comment', id=return_to_model_id,
                                 _query={'target_url': 'consultation',
                                         'parent_id': flattened_comment['comment'].id});
                                 form nested_comment_form">
@@ -137,7 +137,7 @@
                 <!--? Answer for a child comment  -->
                 <div class="row comment-answer-form-container">
                     <metal:block use-macro="layout.macros['answer_comment_form']"
-                             tal:define="add_comment_route_name request.route_url('add_comment', id=consultation.id,
+                             tal:define="add_comment_route_name request.route_url('add_comment', id=return_to_model_id,
                                     _query={'target_url': 'consultation',
                                             'parent_id': flattened_comment['comment'].id});
                                     form nested_comment_form">
@@ -204,7 +204,6 @@
         <div class="bubble-menu">
             <button type="button" data-type="bold">B</button>
             <button type="button" data-type="italic">I</button>
-            <button type="button" data-type="strike">S</button>
         </div>
     </div>
     <input type="hidden" id="${field.id}" name="${field.name}" value="${field.data or ''}"/>