From ac5f842390e0cfe752abded7486d97309af9efae Mon Sep 17 00:00:00 2001 From: Maria Grimaldi Date: Thu, 18 Apr 2024 12:03:32 -0400 Subject: [PATCH 1/5] build: add support for python newer versions in CI GH workflows --- .github/workflows/ci.yml | 4 ++-- setup.py | 7 +++---- src/enmerkar_underscore/__init__.py | 2 +- tox.ini | 7 +++---- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73c4961..4250987 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,8 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: ['3.8'] - toxenv: [django32, django42] + python-version: ['3.8', '3.11', '3.12'] + toxenv: ['django42'] steps: - uses: actions/checkout@v1 diff --git a/setup.py b/setup.py index ce5c344..d4b0433 100644 --- a/setup.py +++ b/setup.py @@ -135,10 +135,9 @@ def get_version(*file_paths): 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Framework :: Django', - 'Framework :: Django :: 2.2', - 'Framework :: Django :: 3.0', - 'Framework :: Django :: 3.1', - 'Framework :: Django :: 3.2', + 'Framework :: Django :: 4.2', ], ) diff --git a/src/enmerkar_underscore/__init__.py b/src/enmerkar_underscore/__init__.py index 66c2b0f..b15d8a4 100644 --- a/src/enmerkar_underscore/__init__.py +++ b/src/enmerkar_underscore/__init__.py @@ -15,7 +15,7 @@ from markey.machine import tokenize, parse_arguments -__version__ = '2.2.1' +__version__ = '2.3.0' def extract(fileobj, keywords, comment_tags, options): """Extracts translation messages from underscore template files. diff --git a/tox.ini b/tox.ini index 24a3764..7615aa0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,18 @@ [tox] -envlist = py38-django{32,42} +envlist = py{38,311,312}-django{42} [testenv] skipsdist = True usedevelop = True deps = -r{toxinidir}/requirements/tox.txt - django32: Django>=3.2,<3.3 django42: Django>=4.2,<4.3 -e. -commands = +commands = python -Wd -m pytest {posargs} [testenv:docs] -commands = +commands = pip install -e {toxinidir} pip install -e {toxinidir}[docs] sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html From d57a51d45c0adcfa40b6129d2528009eeeb1cf55 Mon Sep 17 00:00:00 2001 From: Maria Grimaldi Date: Wed, 24 Apr 2024 14:46:13 -0400 Subject: [PATCH 2/5] build!: remove markey old constraint --- requirements/base.in | 2 +- requirements/base.txt | 14 +++++++------- requirements/dev.txt | 38 ++++++++++++++++++++------------------ requirements/pip-tools.txt | 16 +++++++++------- requirements/pip.txt | 4 ++-- requirements/test.txt | 38 ++++++++++++++++++++------------------ requirements/tox.txt | 36 +++++++++++++++++++----------------- 7 files changed, 78 insertions(+), 70 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 463bf35..88c6640 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -5,5 +5,5 @@ django babel>=1.3 -markey>=0.8,<0.9 +markey enmerkar==0.7.1 diff --git a/requirements/base.txt b/requirements/base.txt index 00f7d68..4e34322 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,13 +4,15 @@ # # make upgrade # -asgiref==3.7.2 +asgiref==3.8.1 # via django babel==2.14.0 # via # -r requirements/base.in # enmerkar -django==3.2.24 +backports-zoneinfo==0.2.1 + # via django +django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -20,10 +22,8 @@ enmerkar==0.7.1 markey==0.8 # via -r requirements/base.in pytz==2024.1 - # via - # babel - # django -sqlparse==0.4.4 + # via babel +sqlparse==0.5.0 # via django -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via asgiref diff --git a/requirements/dev.txt b/requirements/dev.txt index 3abba1c..c040238 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,23 +6,25 @@ # alabaster==0.7.13 # via sphinx -asgiref==3.7.2 +asgiref==3.8.1 # via django babel==2.14.0 # via # -r requirements/base.in # enmerkar # sphinx +backports-zoneinfo==0.2.1 + # via django certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests -coverage[toml]==7.4.2 +coverage[toml]==7.5.0 # via # -r requirements/test.in # pytest-cov # python-coveralls -django==3.2.24 +django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -31,18 +33,20 @@ docutils==0.20.1 # via sphinx enmerkar==0.7.1 # via -r requirements/base.in -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via pytest -execnet==2.0.2 +execnet==2.1.1 # via pytest-cache flake8==7.0.0 # via -r requirements/test.in -idna==3.6 +idna==3.7 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.1 - # via sphinx +importlib-metadata==6.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # sphinx iniconfig==2.0.0 # via pytest jinja2==3.1.3 @@ -53,13 +57,13 @@ markupsafe==2.1.5 # via jinja2 mccabe==0.7.0 # via flake8 -packaging==23.2 +packaging==24.0 # via # pytest # sphinx pep8==1.7.1 # via pytest-pep8 -pluggy==1.4.0 +pluggy==1.5.0 # via pytest pycodestyle==2.11.1 # via flake8 @@ -69,7 +73,7 @@ pyflakes==3.2.0 # pytest-flakes pygments==2.17.2 # via sphinx -pytest==8.0.1 +pytest==8.1.1 # via # -r requirements/test.in # pytest-cache @@ -78,7 +82,7 @@ pytest==8.0.1 # pytest-pep8 pytest-cache==1.0 # via pytest-pep8 -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.in pytest-flakes==4.0.5 # via -r requirements/test.in @@ -87,9 +91,7 @@ pytest-pep8==1.0.6 python-coveralls==2.9.3 # via -r requirements/test.in pytz==2024.1 - # via - # babel - # django + # via babel pyyaml==6.0.1 # via python-coveralls requests==2.31.0 @@ -114,15 +116,15 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlparse==0.4.4 +sqlparse==0.5.0 # via django tomli==2.0.1 # via # coverage # pytest -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via asgiref urllib3==2.2.1 # via requests -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 44c48d9..748bf44 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,15 +4,17 @@ # # make upgrade # -build==1.0.3 +build==1.2.1 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==7.0.1 - # via build -packaging==23.2 +importlib-metadata==6.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # build +packaging==24.0 # via build -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 # via @@ -23,9 +25,9 @@ tomli==2.0.1 # build # pip-tools # pyproject-hooks -wheel==0.42.0 +wheel==0.43.0 # via pip-tools -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index 71954cc..e3ffcc7 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.42.0 +wheel==0.43.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: pip==24.0 # via -r requirements/pip.in -setuptools==69.1.0 +setuptools==69.5.1 # via -r requirements/pip.in diff --git a/requirements/test.txt b/requirements/test.txt index 3abba1c..c040238 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,23 +6,25 @@ # alabaster==0.7.13 # via sphinx -asgiref==3.7.2 +asgiref==3.8.1 # via django babel==2.14.0 # via # -r requirements/base.in # enmerkar # sphinx +backports-zoneinfo==0.2.1 + # via django certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests -coverage[toml]==7.4.2 +coverage[toml]==7.5.0 # via # -r requirements/test.in # pytest-cov # python-coveralls -django==3.2.24 +django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -31,18 +33,20 @@ docutils==0.20.1 # via sphinx enmerkar==0.7.1 # via -r requirements/base.in -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via pytest -execnet==2.0.2 +execnet==2.1.1 # via pytest-cache flake8==7.0.0 # via -r requirements/test.in -idna==3.6 +idna==3.7 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.1 - # via sphinx +importlib-metadata==6.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # sphinx iniconfig==2.0.0 # via pytest jinja2==3.1.3 @@ -53,13 +57,13 @@ markupsafe==2.1.5 # via jinja2 mccabe==0.7.0 # via flake8 -packaging==23.2 +packaging==24.0 # via # pytest # sphinx pep8==1.7.1 # via pytest-pep8 -pluggy==1.4.0 +pluggy==1.5.0 # via pytest pycodestyle==2.11.1 # via flake8 @@ -69,7 +73,7 @@ pyflakes==3.2.0 # pytest-flakes pygments==2.17.2 # via sphinx -pytest==8.0.1 +pytest==8.1.1 # via # -r requirements/test.in # pytest-cache @@ -78,7 +82,7 @@ pytest==8.0.1 # pytest-pep8 pytest-cache==1.0 # via pytest-pep8 -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.in pytest-flakes==4.0.5 # via -r requirements/test.in @@ -87,9 +91,7 @@ pytest-pep8==1.0.6 python-coveralls==2.9.3 # via -r requirements/test.in pytz==2024.1 - # via - # babel - # django + # via babel pyyaml==6.0.1 # via python-coveralls requests==2.31.0 @@ -114,15 +116,15 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlparse==0.4.4 +sqlparse==0.5.0 # via django tomli==2.0.1 # via # coverage # pytest -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via asgiref urllib3==2.2.1 # via requests -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata diff --git a/requirements/tox.txt b/requirements/tox.txt index c71d736..fdf1c09 100644 --- a/requirements/tox.txt +++ b/requirements/tox.txt @@ -6,18 +6,20 @@ # alabaster==0.7.13 # via sphinx -asgiref==3.7.2 +asgiref==3.8.1 # via django babel==2.14.0 # via # -r requirements/base.in # enmerkar # sphinx +backports-zoneinfo==0.2.1 + # via django certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests -coverage[toml]==7.4.2 +coverage[toml]==7.5.0 # via # -r requirements/test.in # pytest-cov @@ -30,18 +32,20 @@ docutils==0.20.1 # via sphinx enmerkar==0.7.1 # via -r requirements/base.in -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via pytest -execnet==2.0.2 +execnet==2.1.1 # via pytest-cache flake8==7.0.0 # via -r requirements/test.in -idna==3.6 +idna==3.7 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.1 - # via sphinx +importlib-metadata==6.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # sphinx iniconfig==2.0.0 # via pytest jinja2==3.1.3 @@ -52,13 +56,13 @@ markupsafe==2.1.5 # via jinja2 mccabe==0.7.0 # via flake8 -packaging==23.2 +packaging==24.0 # via # pytest # sphinx pep8==1.7.1 # via pytest-pep8 -pluggy==1.4.0 +pluggy==1.5.0 # via pytest pycodestyle==2.11.1 # via flake8 @@ -68,7 +72,7 @@ pyflakes==3.2.0 # pytest-flakes pygments==2.17.2 # via sphinx -pytest==8.0.1 +pytest==8.1.1 # via # -r requirements/test.in # pytest-cache @@ -77,7 +81,7 @@ pytest==8.0.1 # pytest-pep8 pytest-cache==1.0 # via pytest-pep8 -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.in pytest-flakes==4.0.5 # via -r requirements/test.in @@ -86,9 +90,7 @@ pytest-pep8==1.0.6 python-coveralls==2.9.3 # via -r requirements/test.in pytz==2024.1 - # via - # babel - # django + # via babel pyyaml==6.0.1 # via python-coveralls requests==2.31.0 @@ -113,15 +115,15 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlparse==0.4.4 +sqlparse==0.5.0 # via django tomli==2.0.1 # via # coverage # pytest -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via asgiref urllib3==2.2.1 # via requests -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata From 90ae4c001ef79d69bc35567ef33b80fe9f8d12d6 Mon Sep 17 00:00:00 2001 From: Maria Grimaldi Date: Wed, 24 Apr 2024 14:51:20 -0400 Subject: [PATCH 3/5] build: add condition for using backports-zoneinfo --- requirements/base.txt | 6 ++++-- requirements/constraints.txt | 4 ++++ requirements/dev.txt | 6 ++++-- requirements/test.txt | 6 ++++-- requirements/tox.txt | 6 ++++-- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 4e34322..8105dab 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -10,8 +10,10 @@ babel==2.14.0 # via # -r requirements/base.in # enmerkar -backports-zoneinfo==0.2.1 - # via django +backports-zoneinfo==0.2.1 ; python_version < "3.9" + # via + # -c requirements/constraints.txt + # django django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt diff --git a/requirements/constraints.txt b/requirements/constraints.txt index b231642..5d33821 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -10,3 +10,7 @@ # Common constraints for edx repos -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + + +# Temporary to Support the python 3.11 Upgrade +backports.zoneinfo;python_version<"3.9" # Newer versions have zoneinfo available in the standard library diff --git a/requirements/dev.txt b/requirements/dev.txt index c040238..a9b1781 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -13,8 +13,10 @@ babel==2.14.0 # -r requirements/base.in # enmerkar # sphinx -backports-zoneinfo==0.2.1 - # via django +backports-zoneinfo==0.2.1 ; python_version < "3.9" + # via + # -c requirements/constraints.txt + # django certifi==2024.2.2 # via requests charset-normalizer==3.3.2 diff --git a/requirements/test.txt b/requirements/test.txt index c040238..a9b1781 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -13,8 +13,10 @@ babel==2.14.0 # -r requirements/base.in # enmerkar # sphinx -backports-zoneinfo==0.2.1 - # via django +backports-zoneinfo==0.2.1 ; python_version < "3.9" + # via + # -c requirements/constraints.txt + # django certifi==2024.2.2 # via requests charset-normalizer==3.3.2 diff --git a/requirements/tox.txt b/requirements/tox.txt index fdf1c09..ca3517d 100644 --- a/requirements/tox.txt +++ b/requirements/tox.txt @@ -13,8 +13,10 @@ babel==2.14.0 # -r requirements/base.in # enmerkar # sphinx -backports-zoneinfo==0.2.1 - # via django +backports-zoneinfo==0.2.1 ; python_version < "3.9" + # via + # -c requirements/constraints.txt + # django certifi==2024.2.2 # via requests charset-normalizer==3.3.2 From 147ebaf7d280b9fdcd6fa9bab1652f188643337d Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 29 Apr 2024 10:23:06 -0400 Subject: [PATCH 4/5] fix: Vendor markey code since the upstream repo is defunct. The README has more info. --- requirements/base.in | 1 - src/enmerkar_underscore/__init__.py | 7 +- src/enmerkar_underscore/vendor/markey/COPYING | 28 ++++ .../vendor/markey/README.rst | 11 ++ .../vendor/markey/__init__.py | 0 .../vendor/markey/_compat.py | 87 ++++++++++++ .../vendor/markey/machine.py | 119 ++++++++++++++++ .../vendor/markey/rules.py | 34 +++++ .../vendor/markey/tests/__init__.py | 0 .../vendor/markey/tests/test_machine.py | 15 ++ .../vendor/markey/tests/test_rules.py | 21 +++ .../vendor/markey/tests/test_tools.py | 96 +++++++++++++ .../vendor/markey/tools.py | 130 ++++++++++++++++++ .../vendor/markey/underscore.py | 54 ++++++++ tox.ini | 1 + 15 files changed, 599 insertions(+), 5 deletions(-) create mode 100644 src/enmerkar_underscore/vendor/markey/COPYING create mode 100644 src/enmerkar_underscore/vendor/markey/README.rst create mode 100644 src/enmerkar_underscore/vendor/markey/__init__.py create mode 100644 src/enmerkar_underscore/vendor/markey/_compat.py create mode 100644 src/enmerkar_underscore/vendor/markey/machine.py create mode 100644 src/enmerkar_underscore/vendor/markey/rules.py create mode 100644 src/enmerkar_underscore/vendor/markey/tests/__init__.py create mode 100644 src/enmerkar_underscore/vendor/markey/tests/test_machine.py create mode 100644 src/enmerkar_underscore/vendor/markey/tests/test_rules.py create mode 100644 src/enmerkar_underscore/vendor/markey/tests/test_tools.py create mode 100644 src/enmerkar_underscore/vendor/markey/tools.py create mode 100644 src/enmerkar_underscore/vendor/markey/underscore.py diff --git a/requirements/base.in b/requirements/base.in index 88c6640..cb5bffb 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -5,5 +5,4 @@ django babel>=1.3 -markey enmerkar==0.7.1 diff --git a/src/enmerkar_underscore/__init__.py b/src/enmerkar_underscore/__init__.py index b15d8a4..f69c9bc 100644 --- a/src/enmerkar_underscore/__init__.py +++ b/src/enmerkar_underscore/__init__.py @@ -1,5 +1,4 @@ import django - from django.template.base import Lexer if django.VERSION[:2] >= (2, 1): @@ -10,10 +9,10 @@ from django.utils.encoding import force_str from enmerkar.extract import extract_django -from markey import underscore -from markey.tools import TokenStream -from markey.machine import tokenize, parse_arguments +from .vendor.markey import underscore +from .vendor.markey.machine import parse_arguments, tokenize +from .vendor.markey.tools import TokenStream __version__ = '2.3.0' diff --git a/src/enmerkar_underscore/vendor/markey/COPYING b/src/enmerkar_underscore/vendor/markey/COPYING new file mode 100644 index 0000000..721e4ce --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/COPYING @@ -0,0 +1,28 @@ +Copyright (C) 2014 Christopher Grebs. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. The name of the author may not be used to endorse or promote + products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/enmerkar_underscore/vendor/markey/README.rst b/src/enmerkar_underscore/vendor/markey/README.rst new file mode 100644 index 0000000..154e7ac --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/README.rst @@ -0,0 +1,11 @@ +These files were copied here from https://github.com/EnTeQuAk/markey which is a +project by the same original owner as the project this repo was forked from +(django-babel-underscore). + +The original project hasn't had updates in 8 years and while a PR was made to +make the same fix upstream (https://github.com/EnTeQuAk/markey/pull/6) we can't +wait for it to get merged before the Redwood release cut ( May 2024). + +To reduce the number of forks we have in play for frontend tech that we have +already deprecated, we copy the bits we need here and make the updates we need +to get support for pythen 3.11 and 3.12. diff --git a/src/enmerkar_underscore/vendor/markey/__init__.py b/src/enmerkar_underscore/vendor/markey/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/enmerkar_underscore/vendor/markey/_compat.py b/src/enmerkar_underscore/vendor/markey/_compat.py new file mode 100644 index 0000000..7e6861e --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/_compat.py @@ -0,0 +1,87 @@ +import sys + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + + +_identity = lambda x: x + + +if not PY2: + text_type = str + string_types = (str,) + integer_types = (int, ) + unichr = chr + + text_to_native = lambda s, enc: s + + iterkeys = lambda d: iter(d.keys()) + itervalues = lambda d: iter(d.values()) + iteritems = lambda d: iter(d.items()) + + from io import StringIO, BytesIO + import pickle + + izip = zip + imap = map + range_type = range + + cmp = lambda a, b: (a > b) - (a < b) + +else: + text_type = unicode + string_types = (str, unicode) + integer_types = (int, long) + + text_to_native = lambda s, enc: s.encode(enc) + unichr = unichr + + iterkeys = lambda d: d.iterkeys() + itervalues = lambda d: d.itervalues() + iteritems = lambda d: d.iteritems() + + from cStringIO import StringIO as BytesIO + from StringIO import StringIO + import cPickle as pickle + + from itertools import izip, imap + range_type = xrange + + cmp = cmp + + +number_types = integer_types + (float,) + + +# Mostly taken from Django. +def force_text(s, encoding='utf-8', errors='strict'): + # Handle the common case first for performance reasons. + if isinstance(s, text_type): + return s + try: + if not isinstance(s, string_types): + if PY3: + if isinstance(s, bytes): + s = text_type(s, encoding, errors) + else: + s = text_type(s) + elif hasattr(s, '__unicode__'): + s = text_type(s) + else: + s = text_type(bytes(s), encoding, errors) + else: + # Note: We use .decode() here, instead of six.text_type(s, encoding, + # errors), so that if s is a SafeBytes, it ends up being a + # SafeText at the end. + s = s.decode(encoding, errors) + except UnicodeDecodeError as e: + if not isinstance(s, Exception): + raise UnicodeDecodeError(s, *e.args) + else: + # If we get to here, the caller has passed in an Exception + # subclass populated with non-ASCII bytestring data without a + # working unicode method. Try to handle this without raising a + # further exception by individually forcing the exception args + # to unicode. + s = ' '.join([force_text(arg, encoding, errors) for arg in s]) + return s diff --git a/src/enmerkar_underscore/vendor/markey/machine.py b/src/enmerkar_underscore/vendor/markey/machine.py new file mode 100644 index 0000000..cf87411 --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/machine.py @@ -0,0 +1,119 @@ +from .rules import include +from ._compat import force_text + + +def iter_rules(x, rules): + for rule in rules[x]: + if rule.__class__ is include: + for item in iter_rules(rule, rules): + yield item + else: + yield rule + + +def tokenize(string, rules, encoding='utf-8'): + string = force_text(string) + + pos = 0 + end = len(string) + stack = [(None, 'everything')] + rule_cache = {} + text_buffer = [] + add_text = text_buffer.append + push = stack.append + flatten = ''.join + + while pos < end: + state = stack[-1][1] + if state not in rule_cache: + rule_cache[state] = list(iter_rules(state, rules)) + for rule in rule_cache[state]: + m = rule.match(string, pos) + if m is not None: + # first flush text that is left in the buffer + if text_buffer: + text = flatten(text_buffer) + if text: + yield 'text', text + del text_buffer[:] + + # now enter the new scopes if entered in a + # non silent way + if rule.enter is not None: + push((rule.enter + '_end', rule.enter)) + yield rule.enter + '_begin', m.group() + + # now process the data + if callable(rule.token): + for item in rule.token(m): + yield item + elif rule.token is not None: + yield rule.token, m.group() + + # now check if we leave something. if the state was + # entered non silent, send a close token. + pos = m.end() + for x in range(rule.leave): + announce, item = stack.pop() + if announce is not None: + yield announce, m.group() + + break + else: + char = string[pos] + add_text(char) + pos += 1 + + # if the text buffer is left filled, we flush it + if text_buffer: + text = flatten(text_buffer) + if text: + yield 'text', text + + # if there are things in the stack left we should + # emit the end tokens + for announce, item in reversed(stack): + if announce is not None: + yield announce, u'' + + +def parse_arguments(stream, end_token): + """ + Helper function for function argument parsing. Pass it a + `TokenStream` and the delimiter token for the argument section and + it will extract all position and keyword arguments as well as + each argument's type info (string literal or not). + + Returns a ``(args, kwargs)`` tuple. + """ + args = [] + kwargs = {} + keywords = [] + while stream.current.type != end_token: + if stream.current.type == 'func_string_arg': + value = stream.current.value.strip() + for char in ['\'', '"']: + if value.startswith(char) and value.endswith(char): + value = value[1: -1] + break + stream.next() + if keywords: + for keyword in keywords: + kwargs[keyword] = value + del keywords[:] + else: + args.append((value, 'func_string_arg')) + elif stream.current.type == 'text': + args.append((stream.current.value, 'text')) + stream.next() + elif stream.current.type == 'func_kwarg': + keywords.append(stream.current.value) + stream.next() + elif stream.current.type == 'func_argument_delimiter': + stream.next() + else: + break + for keyword in keywords: + args.append((keyword, 'func_kwarg')) + + return tuple(args), kwargs diff --git a/src/enmerkar_underscore/vendor/markey/rules.py b/src/enmerkar_underscore/vendor/markey/rules.py new file mode 100644 index 0000000..7ddbca8 --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/rules.py @@ -0,0 +1,34 @@ +import re +from ._compat import izip + + +def bygroups(*args): + """ + Callback creator for bygroup yielding. + """ + return lambda m: izip(args, m.groups()) + + +class include(str): + """ + Tells the lexer to include tokens from another set. + """ + __slots__ = () + + +class ruleset(tuple): + __slots__ = () + + def __new__(cls, *args): + return tuple.__new__(cls, args) + + +class rule(object): + """This represents a parsing rule.""" + __slots__ = ('match', 'token', 'enter', 'leave') + + def __init__(self, regexp, token=None, enter=None, leave=0): + self.match = re.compile(regexp, re.U).match + self.token = token + self.enter = enter + self.leave = leave diff --git a/src/enmerkar_underscore/vendor/markey/tests/__init__.py b/src/enmerkar_underscore/vendor/markey/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/enmerkar_underscore/vendor/markey/tests/test_machine.py b/src/enmerkar_underscore/vendor/markey/tests/test_machine.py new file mode 100644 index 0000000..708709a --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/tests/test_machine.py @@ -0,0 +1,15 @@ +from markey.machine import tokenize, parse_arguments +from markey.tools import TokenStream +from markey.underscore import rules as underscore_rules + + +def test_parse_arguments(): + line = '<% gettext(varible, "some string", str="foo")' + stream = TokenStream.from_tuple_iter(tokenize(line, underscore_rules)) + stream.next() + stream.next() + stream.expect('gettext_begin') + stream.expect('func_name').value + args, kwargs = parse_arguments(stream, 'gettext_end') + assert args == (('varible', 'text'), ('some string', 'func_string_arg'),) + assert kwargs == {'str': 'foo'} diff --git a/src/enmerkar_underscore/vendor/markey/tests/test_rules.py b/src/enmerkar_underscore/vendor/markey/tests/test_rules.py new file mode 100644 index 0000000..2a5f878 --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/tests/test_rules.py @@ -0,0 +1,21 @@ +import re + +from markey.rules import bygroups, rule, ruleset + + +def test_bygroups(): + match = re.match(r'(1)(2)', '12') + assert list(bygroups(('one',))(match)) == [(('one',), '1')] + + +class test_rule(): + r = rule(r'(\d)', 'num', 'do_number') + assert r.match('1').group() == '1' + assert r.token == 'num' + assert r.enter == 'do_number' + assert r.leave == 0 + + +class test_ruleset(): + rs = ruleset(rule(r'(\d)', 'num', 'do_number'),) + assert isinstance(rs, tuple) diff --git a/src/enmerkar_underscore/vendor/markey/tests/test_tools.py b/src/enmerkar_underscore/vendor/markey/tests/test_tools.py new file mode 100644 index 0000000..e52780f --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/tests/test_tools.py @@ -0,0 +1,96 @@ +import sys +from six import StringIO + +import pytest + +from markey.tools import Token, TokenStream, TokenStreamIterator + + +def test_tokenstream(): + s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) + assert s.current == Token('a', 1) + + +def test_tokenstream_from_tuple_iter(): + # from_tuple_iter + s = TokenStream.from_tuple_iter(iter((('a', 1), ('b', 2), ('c', 3)))) + assert s.current == Token('a', 1) + + # iter + assert isinstance(iter(s), TokenStreamIterator) + assert tuple(iter(s)) == (Token('a', 1), Token('b', 2), Token('c', 3)) + + +def test_tokenstream_eof(): + # eof + s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) + assert not s.eof + list(s) + assert s.eof + + +def test_tokenstream_look_push(): + # look, push + s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) + assert s.current == Token('a', 1) + assert s.look() == Token('b', 2) + s.next() + assert s.look() == Token('c', 3) + s.push(Token('b', 2)) + assert s.look() == Token('b', 2) + s.push(Token('e', 4), current=True) + assert s.current == Token('e', 4) + assert s.look() == Token('b', 2) + + +def test_tokenstream_skip_next(): + # skip, next + s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) + s.skip(1) + assert s.current == Token('b', 2) + s.next() + assert s.current == Token('c', 3) + s.push(Token('e', 4)) + assert s.current == Token('c', 3) + s.next() + assert s.current == Token('e', 4) + s.next() + assert s.current == Token('eof', None) + + +def test_tokenstream_expect(): + # expect + s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) + assert s.expect('a') == Token('a', 1) + assert s.expect('b', 2) == Token('b', 2) + pytest.raises(AssertionError, s.expect, 'e') + pytest.raises(AssertionError, s.expect, 'c', 5) + + +def test_tokenstream_test_shift(): + # test + s = TokenStream(iter((Token('a', 1), Token('b', 2), Token('c', 3)))) + assert s.test('a') + s.next() + assert s.test('b', 2) + + # shift + assert s.current == Token('b', 2) + s.shift(Token('f', 5)) + assert s.current == Token('f', 5) + s.next() + assert s.current == Token('b', 2) + + +def test_tokenstream_debug(): + stream = StringIO() + + _original_stdout = sys.stdout + sys.stdout = stream + + try: + s = TokenStream(iter((Token('a', 1),))) + s.debug() + assert stream.getvalue() == "Token(type='a', value=1)\n" + finally: + sys.stdout = _original_stdout diff --git a/src/enmerkar_underscore/vendor/markey/tools.py b/src/enmerkar_underscore/vendor/markey/tools.py new file mode 100644 index 0000000..edef281 --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/tools.py @@ -0,0 +1,130 @@ +import sys +from collections import namedtuple + +#: Represents one token. +Token = namedtuple('Token', ('type', 'value')) + + +class TokenStreamIterator(object): + """ + The iterator for tokenstreams. Iterate over the stream + until the eof token is reached. + """ + + def __init__(self, stream): + self._stream = stream + + def __iter__(self): + return self + + def __next__(self): + token = self._stream.current + if token.type == 'eof': + raise StopIteration() + self._stream.next() + return token + + # Python 2.x + next = __next__ + + +class TokenStream(object): + """ + A token stream wraps a generator and supports pushing tokens back. + It also provides some functions to expect tokens and similar stuff. + + Important note: Do never push more than one token back to the + stream. Although the stream object won't stop you + from doing so, the behavior is undefined. Multiple + pushed tokens are only used internally! + """ + + def __init__(self, generator): + self._next = lambda: next(generator) + self._pushed = [] + self.current = Token('initial', '') + self.next() + + @classmethod + def from_tuple_iter(cls, tupleiter): + return cls(Token(*a) for a in tupleiter) + + def __iter__(self): + return TokenStreamIterator(self) + + @property + def eof(self): + """Are we at the end of the tokenstream?""" + return not bool(self._pushed) and self.current.type == 'eof' + + def debug(self, stream=None): + """Displays the tokenized code on the stream provided or stdout.""" + if stream is None: + stream = sys.stdout + for token in self: + stream.write(repr(token) + '\n') + + def look(self): + """See what's the next token.""" + if self._pushed: + return self._pushed[-1] + old_token = self.current + self.next() + new_token = self.current + self.current = old_token + self.push(new_token) + return new_token + + def push(self, token, current=False): + """Push a token back to the stream (only one!).""" + self._pushed.append(token) + if current: + self.next() + + def skip(self, n): + """Got n tokens ahead.""" + for x in range(n): + self.next() + + def next(self): + """Go one token ahead.""" + if self._pushed: + self.current = self._pushed.pop() + else: + try: + self.current = self._next() + except StopIteration: + if self.current.type != 'eof': + self.current = Token('eof', None) + + def expect(self, type, value=None): + """expect a given token.""" + assert self.current.type == type, "expect failed (%s, %s)" % ( + self.current.type, type) + if value is not None: + assert self.current.value == value or \ + (value.__class__ is tuple and + self.current.value in value), "%s != %s" % (type, value) + try: + return self.current + finally: + self.next() + + def test(self, type, value=Ellipsis): + """Test the current token.""" + return ( + self.current.type == type and + (value is Ellipsis or self.current.value == value or + value.__class__ is tuple and + self.current.value in value)) + + def shift(self, token): + """ + Push one token into the stream. + """ + old_current = self.current + self.next() + self.push(self.current) + self.push(old_current) + self.push(token) + self.next() diff --git a/src/enmerkar_underscore/vendor/markey/underscore.py b/src/enmerkar_underscore/vendor/markey/underscore.py new file mode 100644 index 0000000..4ed9e78 --- /dev/null +++ b/src/enmerkar_underscore/vendor/markey/underscore.py @@ -0,0 +1,54 @@ +# Full functioning underscore template parser. + +from .rules import ruleset, include, rule, bygroups + + +keywords = frozenset(( + 'pluralidx', 'gettext', 'ngettext', 'gettext_noop', 'pgettext', + 'npgettext', '_' +)) + + +rules = { + 'everything': ruleset( + include('inline'), + include('function'), + ), + 'inline': ruleset( + rule(r'<%=', enter='interpolate'), + rule(r'<%-', enter='escape'), + rule(r'<%', enter='evaluate'), + ), + 'inline_with_func': ruleset( + include('inline'), + include('function') + ), + 'function': ruleset( + rule(r'({keywords})\('.format(keywords='|'.join(keywords)), + bygroups('func_name'), enter='gettext') + ), + 'gettext': ruleset( + rule(r'([\"\'])?\)', leave=1), + include('function_call') + ), + 'evaluate': ruleset( + rule(r'%>', leave=1), + include('inline_with_func'), + ), + 'interpolate': ruleset( + rule(r'%>', leave=1), + include('inline_with_func'), + ), + 'escape': ruleset( + rule(r'%>', leave=1), + include('inline_with_func'), + ), + # function calls (parse string arguments and implicit strings) + 'function_call': ruleset( + rule(',', 'func_argument_delimiter'), + rule('\s+', None), + rule(r"(?s)('([^'\\]*(?:\\.[^'\\]*)*)'|" + r'"([^"\\]*(?:\\.[^"\\]*)*)")', 'func_string_arg'), + rule(r'([\w_]+)\s*=', bygroups('func_kwarg')) + ) +} diff --git a/tox.ini b/tox.ini index 7615aa0..8297c54 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ envlist = py{38,311,312}-django{42} skipsdist = True usedevelop = True deps = + setuptools -r{toxinidir}/requirements/tox.txt django42: Django>=4.2,<4.3 -e. From 69fed26491dc204137b8db9f57539ee838b3973a Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 29 Apr 2024 14:10:11 -0400 Subject: [PATCH 5/5] docs: Apply suggestions from code review Include a link to the hash of the upstream code that we used as the base for our vendored code. Co-authored-by: Sarina Canelake --- src/enmerkar_underscore/vendor/markey/README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/enmerkar_underscore/vendor/markey/README.rst b/src/enmerkar_underscore/vendor/markey/README.rst index 154e7ac..4513821 100644 --- a/src/enmerkar_underscore/vendor/markey/README.rst +++ b/src/enmerkar_underscore/vendor/markey/README.rst @@ -4,8 +4,8 @@ project by the same original owner as the project this repo was forked from The original project hasn't had updates in 8 years and while a PR was made to make the same fix upstream (https://github.com/EnTeQuAk/markey/pull/6) we can't -wait for it to get merged before the Redwood release cut ( May 2024). +wait for it to get merged before the Redwood release cut (May 2024). To reduce the number of forks we have in play for frontend tech that we have already deprecated, we copy the bits we need here and make the updates we need -to get support for pythen 3.11 and 3.12. +to get support for Python 3.11 and 3.12. The content was copied from https://github.com/EnTeQuAk/markey/commit/d9fc042b147824164908f00a24da6ce2c26dcd8e before we added modifications to allow for Python 3.11 support.