Milestone 2.0 - User scope views, Discoverable Groups & Patches

.github/labeler.yml

@@ -3,6 +3,9 @@
 api:
   - magpie/api/**/*
 
+cli:
+  - magpie/cli/**/*
+
 # label 'ci' all automation-related steps and files
 ci:
   - .*  # all '.<>' files

.gitignore

@@ -46,6 +46,8 @@ magpie/include
 magpie/lib
 magpie/local
 magpie/ui/swagger-ui/magpie-rest-api.json
+magpie_create_users*.txt
+magpie_delete_users*.txt
 requirements-all.txt
 gunicorn.app.wsgiapp
 error_log.txt

.pylintrc

@@ -76,13 +76,16 @@ disable=C0111,missing-docstring,
         R0915,too-many-statements,
         R1710,inconsistent-return-statements,
         R1721,unnecessary-comprehension,
+        R1725,super-with-arguments,
         W0108,unnecessary-lambda,
+        W0143,comparison-with-callable,
         W0232,no-init,
         W0235,useless-super-delegation,
         W0613,unused-argument,
         W0640,cell-var-from-loop,
         W0703,broad-except,
         W0706,try-except-raise,
+        W0707,raise-missing-from,
         W1508,invalid-envvar-default
 
 # note: following errors are managed via isort tool, ignore false positives
@@ -93,7 +96,10 @@ disable=C0111,missing-docstring,
 # either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member
+enable=
+    c-extension-no-member
+    W0122
+    W0123
 
 
 [REPORTS]
@@ -130,7 +136,10 @@ max-nested-blocks=5
 # inconsistent-return-statements if a never returning function is called then
 # it will be considered as an explicit return statement and no message will be
 # printed.
-never-returning-functions=sys.exit
+never-returning-functions=
+    sys.exit
+    magpie.utils.raise_log
+    magpie.api.exception.raise_http
 
 
 [TYPECHECK]
@@ -365,7 +374,7 @@ function-naming-style=snake_case
 #function-rgx=
 
 # Good variable names which should always be accepted, separated by a comma.
-good-names=i,j,k,v,ex,x,y,z,f,h,db,kw,ns,_
+good-names=i,j,k,v,ex,x,y,z,f,fd,h,db,kw,ns,qs,_
 
 # Include a hint for the correct naming format with invalid-name.
 include-naming-hint=no
@@ -461,7 +470,7 @@ valid-metaclass-classmethod-first-arg=cls
 
 # List of modules that can be imported at any level, not just the top level
 # one.
-allow-any-import-level=
+allow-any-import-level=magpie
 
 # Allow wildcard imports from modules that define __all__.
 allow-wildcard-with-all=no

.travis.yml

@@ -5,25 +5,21 @@ dist: xenial
 os:
   - linux
 python:
-  - "3.6"
-  - "3.5"
   - "2.7"
+  - "3.5"
+  - "3.6"
+  - "3.7"
+  - "3.8"
 cache:
   - pip
-  - directories:
-      - $HOME/conda
-      - $HOME/downloads
 # includes PR when base branch = master
 if: branch = master OR tag IS present
 env:
   global:
-    - CONDA_HOME=$HOME/conda
-    - DOWNLOAD_CACHE=$HOME/downloads
     - secure: lzPPXuRTlqLHl2MCAUuy7ROrewSwPFGGKqbADo7npQ4jqVMAb/TQ5PTzIi4IAIvsv817dNj5WSAd39hsp1zSlE1R2jLtbxGEVqwcjTqSv9VStGascRQnCW1hcoSKYqESweEhtdIcH+uXpV8qnDsUWB6YIIP2VjgWPMfSPZPynFKC0IqZm2sFsqDEJJHr3QM5iN3i3Z9GrEIrBEEpoaf0yak5u8LjqoGSaY8ISPGWipBJYn7i/r/+sZpirFXbZRJmI4ljsaHi0tysNWdANOIf7S2+Dy/oGNn/3Br0OMJ5snFe/n6Uf4GcmPGDc1c86oN0FpGcoNSA+lCUPMASghOGwVqBhMjX+qAZZfzEbCGtr2j/MhcpETMO3S9oqelw4CcbZRXxP2Y20L+KGNOCSt3SC4/HdKj5MWRQGUnIZbT9C2Tpn2wTffpVRoKvhik/6UU8/KT12LSwDhjjrfqLgCEFLqH1nBL2Mopkj/eOwQbZ66nZmthrAF7/c+vaB38LWoqBIZi/fOwo6+kt947d1bW6UX6Z1KnCNLN2KO064Hjn+SHb1gAes0JIXIZE7af1Gti2dKmUL4a5eoYyU5vCLlp2nq5yxEtTUdJKZ1UVnkDqRtSaT0F3FgLypbFqi+qZ6AtygxKqplMjeAGhvVSqZTPqRupP1ac/KBBCvMzG8pkfXWs=
   jobs:
     - TEST_TARGET=test-local    START_TARGET=
     - TEST_TARGET=test-remote   START_TARGET=start
-    - TEST_TARGET=coverage      START_TARGET=
 addons:
   postgresql: "9.6"
 services:
@@ -37,18 +33,20 @@ notifications:
   email: false
 # === pipeline ===
 before_install:
+  # obtain details about environment
   - python -V
   - uname -a
   - lsb_release -a
-  - export PYTHON_VERSION=${TRAVIS_PYTHON_VERSION}
-  - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION}
-  - export CONDA_PREFIX=$HOME/conda/envs/magpie-${TRAVIS_PYTHON_VERSION}
-  - export PATH=${CONDA_HOME}/bin:$PATH
   - hash -r
-  - env
-  - make conda-base
+  # fake conda paths to employ preinstalled virtualenv python
+  - export CONDA_ENV_NAME=python${TRAVIS_PYTHON_VERSION}
+  - export CONDA_ENVS_DIR=${HOME}/virtualenv
+  - export CONDA_PREFIX=${CONDA_ENVS_DIR}/${CONDA_ENV_NAME}
+  - export CONDA_ENV_PATH=${CONDA_PREFIX}
+  - env | sort
+  - make info
   - make install-sys
-  #==== magpie env and constants ===
+  # load magpie env and constants
   - mkdir -p ./env
   - cp -f ./ci/magpie.env ./env/magpie.env
 install:
@@ -59,38 +57,39 @@ before_script:
   - psql -c 'create database magpie;' -U postgres
   - echo ${CONDA_PREFIX}
   - echo ${CONDA_ENV}
-  - export PYTHON_VERSION=${TRAVIS_PYTHON_VERSION}
-  - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION}
-  - export CONDA_PREFIX=${HOME}/conda/envs/magpie-${TRAVIS_PYTHON_VERSION}
-  - export PATH=${CONDA_HOME}/bin:${PATH}
+  - source ./env/magpie.env
   - hash -r
-  - env
+  - env | sort
 stages:
   - check       # run linting checks and don't bother with the rest if invalid
   - test        # use default stage to run job matrix variations
   - smoke-test  # try running the built/packaged docker image
 jobs:
+  allow_failures:
+    # leave versions while they remains easy to maintain
+    - python: "2.7"  # deprecated support (January 2020)
+    - python: "3.5"  # deprecated support (September 2020)
   include:
     # use stages to quick fail faster tests
     # these are extra to default 'test' stage with auto-matrix/env extension
     - stage: check
       name: "Linter Checks"
-      python: "3.6"
+      python: "3.7"
       os: linux
       script: make check
     - stage: check
       name: "Documentation Check"   # verify that build works
-      python: "3.6"
+      python: "3.7"
       os: linux
       script: make docs
     - stage: test
       name: "Coverage"
-      python: "3.6"
+      python: "3.7"
       os: linux
       script: make coverage
     - stage: smoke-test
       name: "Smoke Test"
-      python: "3.6"
+      python: "3.7"
       os: linux
       script: make test-docker
 script:
@@ -101,4 +100,4 @@ script:
 after_success:
   # coverage report
   - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
-  - source ${CONDA_PREFIX}/bin/activate ${CONDA_ENV} && python-codacy-coverage -r reports/coverage.xml
+  - source ${VIRTUAL_ENV}/bin/activate && python-codacy-coverage -r reports/coverage.xml

HISTORY.rst → CHANGES.rst

@@ -1,13 +1,100 @@
+.. explicit references must be used in this file (not references.rst) to ensure they are directly rendered on Github
 .. :changelog:
 
-History
-=======
+Changes
+*******
 
 `Unreleased <https://github.com/Ouranosinc/Magpie/tree/master>`_ (latest)
 ------------------------------------------------------------------------------------
 
 * Nothing yet.
 
+`2.0.0 <https://github.com/Ouranosinc/Magpie/tree/2.0.0>`_ (2020-07-31)
+------------------------------------------------------------------------------------
+
+Features / Changes
+~~~~~~~~~~~~~~~~~~~~~
+* Add ``/ui`` route redirect to frontpage when UI is enabled.
+* Add ``/json`` route information into generated Swagger API documentation.
+* Add tag description into generated Swagger API documentation.
+* Add more usage details to start `Magpie` web application in documentation.
+* Add database migration for new ``discoverable`` column of groups.
+* Allow logged user to update its own information both via API and UI
+  (relates to `#170 <https://github.com/Ouranosinc/Magpie/issues/170>`_).
+* Allow logged user of any access-level to register by itself to ``discoverable`` groups.
+* Change some UI CSS for certain pages to improve table readability.
+* Add UI page to render error details from API responses (instead of default server-side HTML error rendering).
+* Add ``MAGPIE_UI_THEME`` with new default *blue* theme and legacy *green* theme (with few improvements).
+* Add more validation and inputs parameters to update ``Group`` information.
+* Add UI input fields to allow administrator to update group description and group discoverability.
+* Allow combined configuration files (``providers``, ``permissions``, ``users`` and ``groups`` sections) with
+  resolution of inter-references between them. File can be specified with ``MAGPIE_CONFIG_PATH`` environment variable
+  or ``magpie.config_path`` setting (example in ``configs``).
+* Add configurable ``User`` creation parameters upon `Magpie` application startup through configuration files
+  (fixes `#47 <https://github.com/Ouranosinc/Magpie/issues/47>`_ and
+  `#204 <https://github.com/Ouranosinc/Magpie/issues/204>`_).
+* Add disabled checkboxes for UI rendering of non-editable items to avoid user doing operations that will always be
+  blocked by corresponding API validation (relates to `#164 <https://github.com/Ouranosinc/Magpie/issues/164>`_).
+* Add more tests to validate forbidden operations such as update or delete of reserved ``User`` and ``Group`` details.
+* Add active version tag at bottom of UI pages (same version as returned by API ``/version`` route).
+* Enforce configuration parameters ``MAGPIE_SECRET``, ``MAGPIE_ADMIN_USER`` and ``MAGPIE_ADMIN_PASSWORD`` by explicitly
+  defined values (either by environment variable or INI settings) to avoid using defaults for security purposes.
+* Change CLI helper ``create_users`` to ``batch_update_users`` to better represent provided functionalities.
+* Change CLI helper ``register_default_users`` to ``register_defaults`` to avoid confusion on groups also created.
+* Extend CLI ``batch_update_users`` functionality with additional options and corresponding tests.
+* Move all CLI helpers under ``magpie.cli`` and provide more details about them in documentation.
+* Allow unspecified ``group_name`` during user creation request to employ ``MAGPIE_ANONYMOUS_GROUP`` by default
+  (i.e.: created user will have no other apparent group membership since it is always attributed for public access).
+* Change all ``PUT`` requests to ``PATCH`` to better reflect their actual behaviour according to RESTful best practices
+  (partial field updates instead of complete resource replacement and conflict responses on duplicate identifiers).
+* Add support of ``Accept`` header and ``format`` query parameter for all API responses, for content-types variations
+  in either plain text, HTML, XML or JSON (default), and include applicable values in schemas for Swagger generation.
+* Add support of new response content-type as XML (must request using ``Accept`` header or ``format`` query parameter).
+* Add documentation details about different types of ``Permission``, interaction between various `Magpie` models,
+  glossary and other general improvements (relates to `#332 <https://github.com/Ouranosinc/Magpie/issues/332>`_ and
+  `#341 <https://github.com/Ouranosinc/Magpie/issues/341>`_).
+* Add alternative response format for service and service-type paths using ``flatten`` query parameter to obtain a flat
+  list of services instead of nested dictionaries (fixes `#345 <https://github.com/Ouranosinc/Magpie/issues/345>`_).
+* Change pre-existing ``list`` query parameter of user-scoped views to ``flatten`` response format to match new query
+  of service-scoped views.
+* Add ``filtered`` query parameter for user-scoped resources permission listing when request user is an administrator.
+* Obsolete all API routes using ``inherited_permission`` format (deprecated since ``0.7.4``) in favor of equivalent
+  ``permissions?inherited=true`` query parameter modifier.
+* Replace ``inherit`` query parameter wherever applicable by ``inherited`` to match documentation names, but preserve
+  backward compatibility support of old name.
+* Add ``MAGPIE_PASSWORD_MIN_LENGTH`` setting with corresponding validation of field during ``User`` creation and update.
+* Avoid returning ``Service`` entries where user, group or both (according to request path and query options) does not
+  actually have any permission set either directly on them or onto one of their respective children ``Resource``. This
+  avoids unnecessarily exposing all ``Service`` for which the user cannot (or should not) be interacting with anyway.
+* Add ``TWITCHER_HOST`` as alternative configuration parameter to define the service public URL, to have a similar
+  naming convention as other use cases covered by ``MAGPIE_HOST`` and ``PHOENIX_HOST``.
+* Modify ``PHOENIX_PUSH`` to be *disabled* by default to be consistent across all locations where corresponding
+  feature is referenced (startup registration, CLI utility, API requests and UI checkbox option) and because this
+  option is an advanced extension not to be considered as default behavior.
+* Python 2.7 and Python 3.5 marked for deprecation (they remain in CI, but are not required to pass), as both
+  reached their EOL as of January/September 2020.
+
+Bug Fixes
+~~~~~~~~~~~~~~~~~~~~~
+* Fix invalid API documentation of request body for ``POST /users/{user_name}/groups``.
+* Fix `#164 <https://github.com/Ouranosinc/Magpie/issues/164>`_ (forbid *special* users and groups update and delete).
+* Fix `#84 <https://github.com/Ouranosinc/Magpie/issues/84>`_ and
+  `#171 <https://github.com/Ouranosinc/Magpie/issues/171>`_ with additional input validation.
+* Fix `#194 <https://github.com/Ouranosinc/Magpie/issues/194>`_ to render API error responses according to content-type.
+* Fix `#337 <https://github.com/Ouranosinc/Magpie/issues/337>`_ documentation mismatch with previously denied request
+  users since they are now allowed to run these requests with new user-scoped functionalities
+  (`#340 <https://github.com/Ouranosinc/Magpie/issues/340>`_).
+* Fix bug introduced in `0.9.4 <https://github.com/Ouranosinc/Magpie/tree/0.9.4>`_
+  (`4a23a49 <https://github.com/Ouranosinc/Magpie/commit/4a23a497e3ce1dc39ccaf31ba1857fc199d399db>`_) where some
+  API routes would not return the `Allowed Permissions` for children ``Resource`` under ``Service``
+  (only ``Service`` permissions would be filled), or when requesting ``Resource`` details directly.
+* Fix input check to avoid situations where updating ``Resource`` name could cause involuntary duplicate errors.
+* Fix minor HTML issues in mako templates.
+* Fix invalid generation of default ``postgres.env`` file from ``magpie.env.example``.
+  File ``postgres.env.example`` will now be correctly employed as documented.
+* Make environment variable ``PHOENIX_PUSH`` refer to ``phoenix.push`` instead of ``magpie.phoenix_push`` to employ
+  same naming schema as all other variables.
+
 `1.11.0 <https://github.com/Ouranosinc/Magpie/tree/1.11.0>`_ (2020-06-19)
 ------------------------------------------------------------------------------------
 
@@ -213,14 +300,14 @@ Bug Fixes
 
 Bug Fixes
 ~~~~~~~~~~~~~~~~~~~~~
-* Fix a bug in ows_parser_factory that caused query parameters for wps services to be case sensitive.
+* Fix a bug in ``ows_parser_factory`` that caused query parameters for wps services to be case sensitive.
 
 `1.6.1 <https://github.com/Ouranosinc/Magpie/tree/1.6.1>`_ (2019-10-01)
 ------------------------------------------------------------------------------------
 
 Bug Fixes
 ~~~~~~~~~~~~~~~~~~~~~
-* Fix migration script for project-api service type.
+* Fix migration script for ``project-api`` service type.
 
 `1.6.0 <https://github.com/Ouranosinc/Magpie/tree/1.6.0>`_ (2019-09-20)
 ------------------------------------------------------------------------------------

CONTRIBUTING.rst

@@ -78,7 +78,7 @@ Before you submit a pull request, check that it meets these guidelines:
 1. The pull request should include tests.
 2. If the pull request adds functionality, the docs should be updated. Put
    your new functionality into a function with a docstring, and add the
-   feature to the list in `history`_ (under relevant category of section `Unreleased`).
+   feature to the list in `changes`_ (under relevant category of section `Unreleased`).
 3. The tests should work for the specified version of Python for this project.
 
 
@@ -93,6 +93,6 @@ To run a subset of tests::
 
 .. References for this page
 .. _new issue: https://github.com/Ouranosinc/Magpie/issues/new
-.. _history: HISTORY.rst
+.. _changes: CHANGES.rst
 .. _installation: docs/installation.rst
 .. _utilities: docs/utilities.rst

Dockerfile

@@ -17,7 +17,7 @@ RUN chmod 0644 $CRON_DIR/magpie-cron
 
 # install dependencies
 COPY magpie/__init__.py magpie/__meta__.py $MAGPIE_DIR/magpie/
-COPY requirements* setup.py README.rst HISTORY.rst $MAGPIE_DIR/
+COPY requirements* setup.py README.rst CHANGES.rst $MAGPIE_DIR/
 RUN apk update \
     && apk add \
     bash \

Dockerfile.adapter

@@ -10,7 +10,7 @@ LABEL Vendor="CRIM"
 
 ENV MAGPIE_DIR=/opt/local/src/magpie
 COPY magpie/__init__.py magpie/__meta__.py $MAGPIE_DIR/magpie/
-COPY requirements* setup.py README.rst HISTORY.rst $MAGPIE_DIR/
+COPY requirements* setup.py README.rst CHANGES.rst $MAGPIE_DIR/
 
 # install dependencies used by Magpie
 RUN apk update \
@@ -23,7 +23,7 @@ RUN apk update \
         supervisor \
         gcc \
         libffi-dev \
-        python-dev \
+        python3-dev \
         musl-dev \
         postgresql-dev \
     && pip install --no-cache-dir --upgrade pip setuptools \