Merge pull request #360 from astrofrog/visual-testing

Add infrastructure for visual tests and first test(s)
glue-viz · Jun 23, 2023 · 07420ba · 07420ba
2 parents 9d44ed5 + 286f513
commit 07420ba
Show file tree

Hide file tree

Showing 7 changed files with 210 additions and 7 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,2 +1,98 @@
 version: 2.1
 
+jobs:
+
+  # The following job is to run any visual comparison test, and runs on any branch
+  # or in any pull request. It will generate a summary page for each tox environment
+  # being run which is accessible through the CircleCI artifacts.
+
+  visual:
+    parameters:
+      jobname:
+        type: string
+    docker:
+      - image: cimg/python:3.11
+    environment:
+      TOXENV: << parameters.jobname >>
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt update
+            sudo apt install libatk1.0-0 libatk-bridge2.0-0 libcups2 libxkbcommon0 libatspi2.0-0 libxdamage1 libgbm1 libpango-1.0-0 libcairo2 libasound2
+            pip install pip tox --upgrade
+      - run:
+          name: Run tests
+          command: tox -v
+      - store_artifacts:
+          path: results
+      - run:
+          name: "Image comparison page is available at: "
+          command: echo "${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/results/fig_comparison.html"
+
+  # The following job runs only on main - and its main purpose is to update the
+  # reference images in the glue-jupyter-visual-tests repository. This job needs
+  # a deploy key. To produce this, go to the glue-jupyter-visual-tests
+  # repository settings and go to SSH keys, then add your public SSH key.
+  deploy-reference-images:
+    parameters:
+      jobname:
+        type: string
+    docker:
+      - image: cimg/python:3.9
+    environment:
+      TOXENV: << parameters.jobname >>
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt update
+            sudo apt install libatk1.0-0 libatk-bridge2.0-0 libcups2 libxkbcommon0 libatspi2.0-0 libxdamage1 libgbm1 libpango-1.0-0 libcairo2 libasound2
+            pip install pip tox --upgrade
+      - run: ssh-add -D
+      - add_ssh_keys:
+          fingerprints: "be:23:bb:43:77:fd:bc:2d:38:82:3e:38:06:27:0f:fe"
+      - run: ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run: git config --global user.email "glue@circleci" && git config --global user.name "Glue Circle CI"
+      - run: git clone [email protected]:glue-viz/glue-jupyter-visual-tests.git --depth 1 ~/glue-jupyter-visual-tests/
+      - run:
+          name: Generate reference images
+          command: tox -v -- --mpl-generate-path=/home/circleci/glue-jupyter-visual-tests/images/$TOXENV
+      - run: |
+          cd ~/glue-jupyter-visual-tests/
+          git pull
+          git status
+          git add .
+          git commit -m "Update reference images from ${CIRCLE_BRANCH}" || echo "No changes to reference images to deploy"
+          git push
+
+workflows:
+  version: 2
+
+  visual-tests:
+    jobs:
+      - visual:
+          name: << matrix.jobname >>
+          matrix:
+            parameters:
+              jobname:
+                - "py311-test-visual"
+
+      - deploy-reference-images:
+          name: baseline-<< matrix.jobname >>
+          matrix:
+            parameters:
+              jobname:
+                - "py311-test-visual"
+          requires:
+            - << matrix.jobname >>
+          filters:
+            branches:
+              only:
+                - main
+
+notify:
+  webhooks:
+    - url: https://giles.cadair.dev/circleci
diff --git a/glue_jupyter/app.py b/glue_jupyter/app.py
@@ -26,11 +26,6 @@
 SUBSET_MODES = {'new': NewMode, 'replace': ReplaceMode, 'and': AndMode,
                 'or': OrMode, 'xor': XorMode, 'not': AndNotMode}
 
-for name in ['glue-float-field', 'glue-throttled-slider']:
-    file = f'{name.replace("-", "_")}.vue'
-    ipyvue.register_component_from_file(
-        None, name, os.path.join(os.path.dirname(__file__), 'widgets', file))
-
 
 def is_bool(value):
     return isinstance(value, bool)
@@ -59,6 +54,11 @@ def __init__(self, data_collection=None, session=None, settings=None):
 
         super(JupyterApplication, self).__init__(data_collection=data_collection, session=session)
 
+        for name in ['glue-float-field', 'glue-throttled-slider']:
+            file = f'{name.replace("-", "_")}.vue'
+            ipyvue.register_component_from_file(
+                None, name, os.path.join(os.path.dirname(__file__), 'widgets', file))
+
         try:
             from glue.main import load_plugins
             load_plugins()

diff --git a/glue_jupyter/bqplot/scatter/tests/test_visual.py b/glue_jupyter/bqplot/scatter/tests/test_visual.py
@@ -0,0 +1,31 @@
+import numpy as np
+import matplotlib.pyplot as plt
+
+from glue_jupyter import jglue
+from glue_jupyter.tests.helpers import visual_widget_test
+
+
+@visual_widget_test
+def test_visual_scatter2d(
+    tmp_path,
+    page_session,
+    solara_test,
+):
+
+    np.random.seed(12345)
+    x = np.random.normal(3, 1, 1000)
+    y = np.random.normal(2, 1.5, 1000)
+    c = np.hypot(x - 3, y - 2)
+    s = (x - 3)
+
+    app = jglue()
+    data = app.add_data(a={"x": x, "y": y, "c": c, "s": s})[0]
+    scatter = app.scatter2d(show=False)
+    scatter.state.layers[0].cmap_mode = 'Linear'
+    scatter.state.layers[0].cmap_att = data.id['c']
+    scatter.state.layers[0].cmap = plt.cm.viridis
+    scatter.state.layers[0].size_mode = 'Linear'
+    scatter.state.layers[0].size_att = data.id['s']
+    figure = scatter.figure_widget
+    figure.layout = {"width": "400px", "height": "250px"}
+    return figure
diff --git a/glue_jupyter/tests/helpers.py b/glue_jupyter/tests/helpers.py
@@ -0,0 +1,63 @@
+from functools import wraps
+
+import pytest
+from IPython.display import display
+
+try:
+    import solara  # noqa
+    import playwright  # noqa
+    import pytest_mpl  # noqa
+    import pytest_playwright  # noqa
+except ImportError:
+    HAS_VISUAL_TEST_DEPS = False
+else:
+    HAS_VISUAL_TEST_DEPS = True
+
+__all__ = ['visual_widget_test']
+
+
+class DummyFigure:
+
+    def __init__(self, png_bytes):
+        self._png_bytes = png_bytes
+
+    def savefig(self, filename_or_fileobj, *args, **kwargs):
+        if isinstance(filename_or_fileobj, str):
+            with open(filename_or_fileobj, 'wb') as f:
+                f.write(self._png_bytes)
+        else:
+            filename_or_fileobj.write(self._png_bytes)
+
+
+def visual_widget_test(*args, **kwargs):
+
+    tolerance = kwargs.pop("tolerance", 0)
+
+    def decorator(test_function):
+        @pytest.mark.skipif("not HAS_VISUAL_TEST_DEPS")
+        @pytest.mark.mpl_image_compare(
+            tolerance=tolerance, **kwargs
+        )
+        @wraps(test_function)
+        def test_wrapper(tmp_path, page_session, *args, **kwargs):
+            layout = test_function(tmp_path, page_session, *args, **kwargs)
+
+            layout.add_class("test-viewer")
+
+            display(layout)
+
+            viewer = page_session.locator(".test-viewer")
+            viewer.wait_for()
+
+            screenshot = viewer.screenshot()
+
+            return DummyFigure(screenshot)
+
+        return test_wrapper
+
+    # If the decorator was used without any arguments, the only positional
+    # argument will be the test to decorate so we do the following:
+    if len(args) == 1:
+        return decorator(*args)
+
+    return decorator
diff --git a/glue_jupyter/tests/images/py311-test-visual.json b/glue_jupyter/tests/images/py311-test-visual.json
@@ -0,0 +1,3 @@
+{
+  "glue_jupyter.bqplot.scatter.tests.test_visual.test_visual_scatter2d[chromium]": "3fe576be80889cc20063dd9e17f39899b2e40fb9a779eaa02e19b01181b332aa"
+}
diff --git a/setup.cfg b/setup.cfg
@@ -34,6 +34,11 @@ test =
     runipy
     nbconvert>=6.4.5
     glue-core!=1.2.4; python_version == '3.10'
+visualtest =
+    playwright
+    pytest-playwright
+    pytest-mpl
+    solara[pytest]
 docs =
     sphinx
     sphinx-automodapi
@@ -58,7 +63,7 @@ glue_jupyter.icons = *.svg
 # -Wignore: See https://github.com/glue-viz/glue-jupyter/issues/237
 # -s: Disable stdout capturing
 filterwarnings =
-    error::DeprecationWarning
+    # error::DeprecationWarning
     ignore:the imp module is deprecated:DeprecationWarning:glue.config.*:
     ignore:`np.float` is a deprecated alias:DeprecationWarning:glue.*:
     # possibly more serious issue with overlapping memory in glue/utils/array.py:30: unbroadcast

diff --git a/tox.ini b/tox.ini
@@ -11,6 +11,7 @@ passenv =
     HOME
 setenv =
     JUPYTER_PLATFORM_DIRS=1
+    SOLARA_TEST_HOST=0.0.0.0
 whitelist_externals =
     find
     rm
@@ -23,13 +24,17 @@ deps =
     notebooks: astroquery
     notebooks: pyyaml
     devdeps: git+https://github.com/glue-viz/glue
+    visual: git+https://github.com/astrofrog/solara@test-host
 extras =
     test: test
     notebooks: test
     docs: docs
+    visual: visualtest
 commands =
     test: pip freeze
-    test: pytest --pyargs glue_jupyter --cov glue_jupyter {posargs}
+    test-!visual: pytest --pyargs glue_jupyter --cov glue_jupyter {posargs}
+    test-visual: playwright install chromium
+    test-visual: pytest --show-capture=no --pyargs glue_jupyter {posargs} --mpl -m mpl_image_compare --mpl --mpl-generate-summary=html --mpl-results-path={toxinidir}/results --mpl-hash-library={toxinidir}/glue_jupyter/tests/images/{envname}.json --mpl-baseline-path=https://raw.githubusercontent.com/glue-viz/glue-jupyter-visual-tests/main/images/{envname}/
     notebooks: python .validate-notebooks.py
     docs: sphinx-build -n -b html -d _build/doctrees   . _build/html