diff --git a/.env b/.env index 0380f3ac..71e479f4 100644 --- a/.env +++ b/.env @@ -18,13 +18,20 @@ APP_VERSION=V.0.1 APP_DEBUG=1 # What is the enviroment type you want to use for local production? (choose between dec,stag,prod, acce or test) APP_ENV=dev +# We use a build to tag images, this is swithced to the version on master and to env on other branches +APP_BUILD=dev # The description for this api APP_DESCRIPTION='Naast deze JSON rest API is er ook een [graphql](/graphql) interface beschikbaar.' +# The urls on wich this api is available +TRUSTED_PROXIES=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 +TRUSTED_HOSTS='^(.+\.)?conduction\.nl$|^(.+\.)?huwelijksplanner\.online$|^(.+\.)?larping\.eu$|^(.+\.)?common-ground\.nl$|^(.+\.)?trouwplanner\.online$|^(.+\.)?zaakonline\.nl$|localhost' + ################################################## # Documentation settings ################################################## +APP_DOMAIN=conduction.nl APP_DEMO=pc.zaakonline.nl APP_REPRO=https://github.com/ConductionNL/Proto-component-commonground diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 529514cf..4f8b430e 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -21,24 +21,38 @@ jobs: - uses: actions/checkout@v1 - name: Pulling old images, if any run: docker-compose pull --ignore-pull-failures + - name: Setting APP_NAME + run: | + export NAME=$(grep APP_NAME= .env | cut -d '=' -f2) + echo ::set-env name=APP_NAME::$NAME + - name: Print app name + run: echo "APP_NAME = $APP_NAME" - name: Setting APP_ENV to dev run: | echo ::set-env name=APP_ENV::dev + echo ::set-env name=APP_BUILD::dev echo "set APP_ENV to $APP_ENV" - name: Setting APP_ENV to prod if: contains( github.ref, 'master' ) || contains( github.base_ref, 'master' ) run: | echo ::set-env name=APP_ENV::prod echo "set APP_ENV to $APP_ENV" + - name: Set APP_BUILD to APP_VERSION + if: contains( github.ref, 'master' ) + run: | + export VERSION=$(grep APP_VERSION= .env | cut -d '=' -f2) + echo ::set-env name=APP_BUILD::$VERSION + echo "set APP_BUILD to $APP_BUILD" - name: Setting APP_ENV to stag if: contains( github.ref, 'staging' ) || contains( github.base_ref, 'staging' ) run: | echo ::set-env name=APP_ENV::stag + echo ::set-env name=APP_BUILD::stag echo "set APP_ENV to $APP_ENV" - name: Print definitive APP_ENV run: echo "APP_ENV is now $APP_ENV" - name: Build the Docker image - run: docker-compose build --pull --build-arg APP_ENV=$APP_ENV + run: docker-compose build --pull --build-arg APP_ENV=$APP_ENV --build-arg APP_BUILD=$APP_BUILD - name: Run the docker image run: docker-compose up -d - name: Taking some sleep @@ -77,5 +91,38 @@ jobs: if: (contains( github.ref, 'master' ) || contains( github.ref, 'staging' ) || contains( github.ref, 'development' )) && steps.kubeconfig.outputs.success == 'true' run: helm init --upgrade --kubeconfig="kubeconfig.yaml" - name: Deploy through helm + id: helm-install if: (contains( github.ref, 'master' ) || contains( github.ref, 'staging' ) || contains( github.ref, 'development' )) && steps.kubeconfig.outputs.success == 'true' - run: helm upgrade pc-$APP_ENV ./api/helm --kubeconfig="kubeconfig.yaml" --namespace=$APP_ENV --set settings.env=$APP_ENV,settings.debug=1,settings.loadbalancerEnabled=true + run: helm upgrade $APP_NAME-$APP_ENV ./api/helm --kubeconfig="kubeconfig.yaml" --namespace=$APP_ENV --set settings.env=$APP_ENV,settings.debug=1 + - name: Install through helm + if: failure() + run: helm install --name $APP_NAME-$APP_ENV ./api/helm --kubeconfig="kubeconfig.yaml" --namespace=$APP_ENV --set settings.env=$APP_ENV,settings.debug=1 + - name: Rollout new containers + if: (contains( github.ref, 'master' ) || contains( github.ref, 'staging' ) || contains( github.ref, 'development' )) && steps.kubeconfig.outputs.success == 'true' && success() + run: | + kubectl rollout restart deployment/$APP_NAME-php --kubeconfig="kubeconfig.yaml" --namespace=$APP_ENV + kubectl rollout restart deployment/$APP_NAME-nginx --kubeconfig="kubeconfig.yaml" --namespace=$APP_ENV + kubectl rollout restart deployment/$APP_NAME-varnish --kubeconfig="kubeconfig.yaml" --namespace=$APP_ENV + - name: Export release code + if: (success() || failure()) + id: releasecode + run: | + export RELEASE=$APP_BUILD-$(git rev-parse --short $GITHUB_SHA) + echo "##[set-output name=releasename]$RELEASE" + - name: Print release name + if: (success() || failure()) + run: echo $RELEASENAME + env: + RELEASENAME: ${{ steps.releasecode.outputs.releasename }} + - name: Create Release + if: contains( github.ref, 'master' ) && steps.kubeconfig.outputs.success == 'true' && ( success() || failure() ) + id: create_release + uses: actions/create-release@v1 + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ steps.releasecode.outputs.releasename }} + release_name: ${{ steps.releasecode.outputs.releasename }} + draft: false + prerelease: false diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 63cda310..ced0c1eb 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,8 +1,8 @@ - - + + @@ -226,6 +353,23 @@ - \ No newline at end of file diff --git a/DESIGN.md b/DESIGN.md index 34ff5696..cffb8595 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -14,10 +14,10 @@ Welcome, you are currently viewing the design decisions for the proto component. - [Search](#search) - [Queries](#queries) - [Extending](#extending) -- [Timetravel](#timetravel) +- [Time travel](#timetravel) - [Archivation](#archivation) -- [Audittrail](#audittrail) -- [Healthchecks](#healthchecks) +- [Audit trail](#audittrail) +- [Health checks](#healthchecks) - [Notifications](#notifications) - [Authentication](#authentication) - [Authorization](#authorization) @@ -29,14 +29,14 @@ Welcome, you are currently viewing the design decisions for the proto component. *Implementation choices* - [Api Versioning](#api-versioning) -- [Environments and namespacing](#environments-and-namespacing) +- [Environments and name spacing](#environments-and-namespacing) - [Domain Build-up and routing](#domain-build-up-and-routing) - [Container Setup](#container-setup) The European factor ------- -The proto-component isn't just a Dutch Component, it is in essence a Dutch translation of European components, nowhere is this more obvious than in the core code. Our component is based on [API Platform](https://api-platform.com/) an API specific version of the symfony framework. This framework is build by the lovely people of []() and is build with support of the European Commission trough the [EU-FOSSA Hackathon](https://ec.europa.eu/info/news/first-eu-fossa-hackathon-it-happened-2019-may-03_en) and Digital Ocean trough [Hacktoberfest](https://hacktoberfest.digitalocean.com/). +The proto-component isn't just a Dutch Component, it is in essence a Dutch translation of European components, nowhere is this more obvious than in the core code. Our component is based on [API Platform](https://api-platform.com/) an API specific version of the symfony framework. This framework is build by the lovely people of [Les Tilleuls](https://les-tilleuls.coop/en) and is build with support of the European Commission trough the [EU-FOSSA Hackathon](https://ec.europa.eu/info/news/first-eu-fossa-hackathon-it-happened-2019-may-03_en) and Digital Ocean trough [Hacktoberfest](https://hacktoberfest.digitalocean.com/). But it doesn't just end there. The [varnish container](https://hub.docker.com/r/eeacms/varnish/) that we use to speed up the API response is build and maintained by [EEA]() (The European Environment Agency) and the development team at conduction itself is attached to the [Odyssey program](https://www.odyssey.org/) and originated from the [startupinresidence](https://startupinresidence.com/) program. @@ -46,10 +46,12 @@ On standards and standardization ------- The specific goal of the proto component (which this current code base is a version of) is to provide a common architecture for common ground components. As such the common ground principles are leading in design choices, and within those principles international compliancy and technological invocation is deemed most important. **We do not want to make concessions to the current infrastructure.** As such the component might differ on [NL API Strategie](https://docs.geostandaarden.nl/api/API-Strategie), [NORA](https://www.noraonline.nl/wiki/Standaarden), [vng.cloud](https://zaakgerichtwerken.vng.cloud/themas/index) and or other standards if they are deemed incompatible or out of line with (inter)national standards and or good practices. -Unfortunatly (inter)national standards standards can be conflicting. We therefore prioritize standards on two grounds +Unfortunately (inter)national standards can be conflicting. We therefore prioritize standards on several grounds -- International standards are put before local standards -- Standards caried by a standard organisation (like ISO, W3C etc) at put before floating standards (like RFC's) wichs are put before industraty standards, good practices and so on. +- International is put before local +- Standards carried by a standard organization (like ISO, W3C etc) at put before floating standards (like RFC's) wichs are put before industry standards, good practices and so on. + +So if for instance a **local** standard is out of line with an **international** good practice we follow the international good practice. ### Commonground specific standards @@ -57,30 +59,30 @@ This component was designed in line with the [NL API Strategie](https://docs.geo ## NL API Strategie -The [NL API Strategie](https://docs.geostandaarden.nl/api/API-Strategie) takes a special place in this component, it is designed as a set of guidelines for API's for the dutch landscape. As such we follow it as close as posible. It dos however contains inconsistenies with both international standards and good practices. On those items we do not follow the norm but consider it our duty to try to change the norm. +The [NL API Strategie](https://docs.geostandaarden.nl/api/API-Strategie) takes a special place in this component, it is designed as a set of guidelines for API's for the Dutch landscape. As such we follow it as close as possible. It dos however contains inconsistencies with both international standards and good practices. On those items we do not follow the norm but consider it our duty to try to change the norm. ** We implement ** api-01, api-02, api-03, api-05, api-06, api-10, api-11, api-12, api-13,api-14, api-16, api-18, api-19, api-20, api-21, api-22, api-23, api-24, api-25, api-26, api-27, api-28, api-29, api-30, api-33, api-34, api-35, api-42 ** We want to implement ** -- [api-14](https://docs.geostandaarden.nl/api/API-Strategie/#api-14) Use OAuth 2.0 for authorisation +- [api-14](https://docs.geostandaarden.nl/api/API-Strategie/#api-14) Use OAuth 2.0 for authorization ** We do not implement ** - [api-04](https://docs.geostandaarden.nl/api/API-Strategie/#api-04) Define interfaces in Dutch unless there is an official English glossary (see [english](#english)) -- [api-09](https://docs.geostandaarden.nl/api/API-Strategie/#api-09) Implement custom representation if supported see [fields](#fields)) +- [api-09](https://docs.geostandaarden.nl/api/API-Strategie/#api-09) Implement custom representation if supported (see [fields](#fields)) - [api-17](https://docs.geostandaarden.nl/api/API-Strategie/#api-17) Publish documentation in Dutch unless there is existing documentation in English or there is an official English glossary (see [english](#english)) - [api-31](https://docs.geostandaarden.nl/api/API-Strategie/#api-31) Use the query parameter sorteer to sort (see [ordering](#ordering)) - [api-32](https://docs.geostandaarden.nl/api/API-Strategie/#api-32) Use the query parameter zoek for full-text search (see [search](#search)) - [api-36](https://docs.geostandaarden.nl/api/API-Strategie/#api-36) Provide a POST endpoint for GEO queries (see [queries](#queries)) - [api-37](https://docs.geostandaarden.nl/api/API-Strategie/#api-37) Support mixed queries at POST endpoints available (see [queries](#queries)) -*[api-38](https://docs.geostandaarden.nl/api/API-Strategie/#api-38) Put results of a global spatial query in the relevant geometric context (see [queries](#queries)) -- +- [api-38](https://docs.geostandaarden.nl/api/API-Strategie/#api-38) Put results of a global spatial query in the relevant geometric context (see [queries](#queries)) + -** We doubt or havn't made a choice yet about** +** We doubt or haven’t made a choice yet about** -- [api-15](https://docs.geostandaarden.nl/api/API-Strategie/#api-15) Use PKIoverheid certificates for access-restricted or purpose-limited API authentication +- [api-15](https://docs.geostandaarden.nl/api/API-Strategie/#api-15) Use PKI overheid certificates for access-restricted or purpose-limited API authentication - [api-39](https://docs.geostandaarden.nl/api/API-Strategie/#api-39) Use ETRS89 as the preferred coordinate reference system (CRS) - [api-40](https://docs.geostandaarden.nl/api/API-Strategie/#api-40) Pass the coordinate reference system (CRS) of the request and the response in the headers - [api-41](https://docs.geostandaarden.nl/api/API-Strategie/#api-41) Use content negotiation to serve different CRS @@ -88,19 +90,19 @@ api-01, api-02, api-03, api-05, api-06, api-10, api-11, api-12, api-13,api-14, a NLX ------- We implement the [NLX system](https://docs.nlx.io/understanding-the-basics/introduction/) as part of the basic commonground infrastructure, as such nlx headers are used in the internal logging. -The following X-NLX headers have been implemented for that reason `X-NLX-Logrecord-ID`,`X-NLX-Request-Process-Id`,`X-NLX-Request-Data-Elements` and `X-NLX-Request-Data-Subject`, these are tied to the internal audit trail (see audit trail for more information), and `X-Audit-Toelichting` (from the ZGW APIs) is implemented as `X-Audit-Clarification` - -We do not use other NLX headers since they (conform to the [NLX schema](https://docs.nlx.io/further-reading/transaction-logs/))wil not reach the provider. Please note that the use of nlx is optional. The component can be used without NLX. In that case the `X-NLX` header should be set to false, the `X-NLX-Logrecord-ID` should be provided with an log record designd by the client application to be retracable to a unique user and action. Other headers still aplly. +The following X-NLX headers have been implemented for that reason `X-NLX-Logrecord-ID`,`X-NLX-Request-Process-Id`,`X-NLX-Request-Data-Elements` and `X-NLX-Request-Data-Subject`, these are tied to the internal audit trail (see audit trail for more information), and `X-Audit-Toelichting` (from the ZGW APIs) is implemented as `X-Audit-Clarification`. We do not use other NLX headers since they (conform to the [NLX schema](https://docs.nlx.io/reference-information/transaction-log-headers/)) wil not reach the provider. We strongly discourage the use of the `X-NLX-Request-Data-Subject` header as it might allow private data (such as BSNs) to show up in logging. +Please note that the use of nlx is optional. The component can be used without NLX. In that case set `X-NLX-Logrecord-ID` to false and provide (the normaly ignored) fields `X-NLX-Requester-User-Id`, `X-NLX-Request-Application-Id`, `X-NLX-Request-Subject-Identifier`, `X-NLX-Requester-Claims` and `X-NLX-Request-User` as if you are making an NLX call. This provides the API with enough credentials to make an complete audit trail. It also provides an easy implementation route to NLX since the only thing that would need to be changed at a later time is making you call to an nlx outway instead of the API directly. + English ------- The [NL API Standard](https://geonovum.github.io/KP-APIs/#api-04-define-interfaces-in-dutch-unless-there-is-an-official-english-glossary) describes that there is a preference for Dutch in API documentation. > Define resources and the underlying entities, fields and so on (the information model ad the external interface) in Dutch. English is allowed in case there is an official English glossary. -We view this as a breach with good coding practice and international coding standards, all documentation and code is therefore supplied in English. We do however provide transaltion (or i18n) support. +We view this as a breach with good coding practice and international coding standards, all documentation and code is therefore supplied in English. We do however provide translation (or i18n) support. Fields ------- @@ -108,49 +110,24 @@ A part of the [haal centraal](https://raw.githubusercontent.com/VNG-Realisatie/H Search ------- -As part of [api-32](https://docs.geostandaarden.nl/api/API-Strategie/#api-32) a `zoeken` query has been itroduced that can handle wildcards. This breaks best practice, first of allest practice is a `search` query parameter (see also the nodes on [English](#english)). Secondly wildcards are a sql concept, not a webconcept, they are also a rather old concept severly limiting the search options provided. Instead the [regeular expresion standard](https://en.wikipedia.org/wiki/Regular_expression) should be used. +As part of [api-32](https://docs.geostandaarden.nl/api/API-Strategie/#api-32) a `zoeken` query has been introduced that can handle wildcards. This breaks best practice, first of allest practice is a `search` query parameter (see also the nodes on [English](#english)). Secondly wildcards are a sql concept, not a webconcept, they are also a rather old concept severely limiting the search options provided. Instead the [regular expression standard](https://en.wikipedia.org/wiki/Regular_expression) should be used. __solution__ We implement a `search` query parameter on resource collections, that filters with regex. Queries ------- -In several examples of the nl apistrategie we see query parameters being atached to post requests. This is unusual in the sence that sending query strings allong with a post is ocnsiderd bad practice (becouse query parameters end up as part of an url and are therfore logged by servers). But is is technically posile folowing RFC 3986. The real pain is that in the NL api-stratgie the POST requests seems to be used to search, ot in other words GET data. This is where compliance with HTTP (1.1) breaks. +In several examples of the nl api strategie we see query parameters being attached to post requests. This is unusual in the sence that sending query strings along with a post is considered bad practice (because query parameters end up as part of an url and are therefore logged by servers). But it is technically possible folowing RFC 3986. The real pain is that in the NL api-stratgie the POST requests seems to be used to search, ot in other words GET data. This is where compliance with HTTP (1.1) breaks. __solution__ We do not implement a query endpoint on post requests. - -Domain Build-up and routing -------- -By convention the component assumes that you follow the common ground domain name build up, meaning {environment}.{component}.{rest of domain}. That means that only the first two url parts are used for routing. It is also assumed that when no environment is supplied the production environment should be offered E.g. a proper domain for the production API of the verzoeken registratie component would be prod.vrc.zaakonline.nl but it should also be reachable under vrc.zaakonline.nl. The proper location for the development environment should always be dev.vrc.zaakonlin.nl - -Environments and namespacing -------- -We assume that for that you want to run several environments for development purposes. We identify the following namespaces for support. -- prod (Production) -- acce (Acceptation) -- stag (Staging) -- test (Testing) -- dev (Development) - -Because we base the common ground infrastructure on kubernetes, and we want to keep a hard separation between environment we also assume that you are using your environment as a namespace - -Symfony library management gives us the option to define the libraries on a per environment base, you can find that definition in the [bundle config](api/config/bundles.php) - -Besides the API environments the component also ships with additional tools/environments but those are not meant to be deployed -- client (An react client frontend) -- admin (An read admin interface) - -On the local development docker deploy the client environment is used as default instead of the production version of the api. - Api Versioning ------- -As per [landelijke API-strategie.](https://geonovum.github.io/KP-APIs/#versioning) major versions in endpoint minor versions in header, for this the `API-Version` is used (instead of the `api-version` header used in haal centraal) - +As per [landelijke API-strategie.](https://geonovum.github.io/KP-APIs/#versioning) we provide/ask major versions in the endpoint and minor versions in header, for this the `API-Version` is used (instead of the `api-version` header used in haal centraal) __solution__ -The fields parameter and functionality has been implemented as an array, and should be used that way. We do howver support an comma separted value list. +We implement both endpoint and header versioning Extending ------- @@ -161,7 +138,7 @@ The extend parameter has been implemented as an array Archivation ------- -There is a need (by law) for archivation, meaning that we should only keep resources for a fixed amount of time and delete them there afther. In line with the extending and fields principle whereby we only want resource properties that we need when we needid, it is deemded good practice make a sub resource of the archivation properties. For the archivation proterties the [zgw](https://zaken-api.vng.cloud/api/v1/schema/#operation/zaak_list) is followed and translated to englisch. +There is a need (by law) for archivation, meaning that we should only keep resources for a fixed amount of time and delete them thereafter. In line with the extending and fields principle whereby we only want resource properties that we need when we needed, it is deemed good practice make a sub resource of the archivation properties. For the archivation properties the [zgw](https://zaken-api.vng.cloud/api/v1/schema/#operation/zaak_list) is followed and translated to englisch. ```json @@ -173,15 +150,15 @@ There is a need (by law) for archivation, meaning that we should only keep resou } ``` -This gives us an intresting thought, acording to [NL API Strategie](https://docs.geostandaarden.nl/api/API-Strategie/#api-10-implement-operations-that-do-not-fit-the-crud-model-as-sub-resources) subresources should have there own endpoint. Therefore we could use a archive sub of a difrend object for archivation rules e.g. /zaken/{uuid}/archivation for a verzoek. This in itself leads credence to the thought that archivation should have its own central crud api. +This gives us an interesting thought, according to [NL API Strategie](https://docs.geostandaarden.nl/api/API-Strategie/#api-10-implement-operations-that-do-not-fit-the-crud-model-as-sub-resources) sub resources should have their own endpoint. Therefore we could use a archive sub of a different resource for archivation rules e.g. /zaken/{uuid}/archivation for a verzoek. This in itself leads credence to the thought that archivation should have its own central crud api. Audittrail ------- For audittrail we use the base mechanism as provided by [vng.cloud](https://zaakgerichtwerken.vng.cloud/themas/achtergronddocumentatie/audit-trail), we do however diver on some key point, -- Personal data schould never be part of a log, therefore only the user id with the client should be logged (insted of the name) +- Personal data should never be part of a log, therefore only the user id with the client should be logged (instead of the name) - Besides an endpoint per resource there should be a general enpoint to search all audit trials of a component -- [Timetravel](#timetravel) in combinaition with objects versioning makes the return of complete objects unnecesary. But an auditrail endpoint should support the [extend](#extending) functionalitiy to provide the option of obtaining complete objects. +- [Time travel](#timetravel) in combination with objects versioning makes the return of complete objects unnecessary. But an audit rail endpoint should support the [extend](#extending) functionality to provide the option of obtaining complete objects. __solution__ @@ -191,17 +168,18 @@ Healthchecks ------- From [issue 154](https://github.com/VNG-Realisatie/huwelijksplanner/issues/154) -For healthc +For healthchecks we use the health-json principle (or json-health to stay in line with json-ld and json-hal). This means the any endpoint `should` be capable of providing health information concerning that endpoint and services behind it. __solution__ +The use of a `Content-Type: application/health+json` header returns an health json schema. Notifications ------- -For notifications we do not YET use the current [ZGW standard](https://zaakgerichtwerken.vng.cloud/themas/achtergronddocumentatie/notificaties) since there is an [dicusion](https://github.com/VNG-Realisatie/gemma-zaken/issues/1427#issuecomment-549272696) about the posible insecurity of sending properties or data objects along with a notification. It also dosn't follow the [web standard](https://www.w3.org/TR/websub/). We wait for the conclusion of that discusion before making an implementation. +For notifications we do not YET use the current [ZGW standard](https://zaakgerichtwerken.vng.cloud/themas/achtergronddocumentatie/notificaties) since there is an [dicusion](https://github.com/VNG-Realisatie/gemma-zaken/issues/1427#issuecomment-549272696) about the possible insecurity of sending properties or data objects along with a notification. It also doesn’t follow the [web standard](https://www.w3.org/TR/websub/). We wait for the conclusion of that discussion before making an implementation. __solution__ -In compliance with [w3.org](https://www.w3.org/TR/websub/) each endpoint returns an header containing an subscribtion url. That can be used in acordanse with the application to subscribe to both individual objects as collections. whereby collections serve as 'kanalen'. We aim to implement the ZGW notificatie component, but feel that further features on that component would be required to make to be fully suported. We will suply feature requests per issue to support this effort. +In compliance with [w3.org](https://www.w3.org/TR/websub/) each endpoint `should` returns an header containing an subscription url. That can be used in accordance with the application to subscribe to both individual objects as collections whereby collections serve as 'kanalen'. We aim to implement the ZGW notificatie component, but feel that further features on that component would be required to make to be fully supported. We will supply feature requests per issue to support this effort. Authentication ------- @@ -210,17 +188,18 @@ __solution__ Authorization ------- -We implement user scopes as per [vng.cloud](https://zaakgerichtwerken.vng.cloud/themas/achtergronddocumentatie/autorisatie-scopes) standard. But see problems with how the scopes are defined and named, and consider the general setup to be to focused on ZGW (including Dutch naming, zgw specific fields like maxVertrouwlijkheid and a lack of CRUD thinking). There is a further document concerning [Authentication and Authorization](https://zaakgerichtwerken.vng.cloud/themas/achtergronddocumentatie/authenticatie-autorisatie) that details how we should authenticate users and give them scopes. We agree with the principles of the document on application based authorization and the use of JWT tokens. But disagree on some key technical aspect. Most important being that the architecture doesn't take into consideration the use of one component by several organizations at once. Or scopese per property. +We implement user scopes as per [vng.cloud](https://zaakgerichtwerken.vng.cloud/themas/achtergronddocumentatie/autorisatie-scopes) standard. But see problems with how the scopes are defined and named, and consider the general setup to be to focused on ZGW (including Dutch naming, zgw specific fields like maxVertrouwlijkheid and a lack of CRUD thinking). There is a further document concerning [Authentication and Authorization](https://zaakgerichtwerken.vng.cloud/themas/achtergronddocumentatie/authenticatie-autorisatie) that details how we should authenticate users and give them scopes. We agree with the principles of the document on application based authorization and the use of JWT tokens. But disagree on some key technical aspect. Most important being that the architecture doesn't take into consideration the use of one component by several organizations at once. Or scopes per property. __solution__ -No solution as of yet, so there is no implementation of Authorization or Scopes. We aim to implement the ZGW authorisatie component, but feel that further features on that component would be required to make to be fully suported. We will suply feature requests per issue to support this effort. +No solution as of yet, so there is no implementation of Authorization or Scopes. We aim to implement the ZGW authorisatie component, but feel that further features on that component would be required to make to be fully supported. We will supply feature requests per issue to support this effort. Timetravel ------- A part of the [haal centraal](https://raw.githubusercontent.com/VNG-Realisatie/Haal-Centraal-BRP-bevragen/master/api-specificatie/Bevraging-Ingeschreven-Persoon/openapi.yaml) the concept of timetravel has been introduced, as in getting the version of an object as it was on a given date. For this the `geldigop` [see the docs](file:///C:/Users/ruben/Desktop/doc_gba_historie.html#operation/getBewoningen) header is used. In addition the `geldigvan` and `geldigtot` are introduced as collection filters. -The commonground proto componant natively supports time traveling on all entities that are annotaded with the @Gedmo\Loggable, this is done by adding the ?validOn=[date] query to a request, date can either be a datetime or datedatime string. Any value supported by php's [strtotime()](https://www.php.net/manual/en/function.strtotime.php) is supported. Keep in mind that this returns the entity a as it was valid on that time or better put, the last changed version BEFORE that moment. To get a complete list of all changes on a item the /audittrail endpoint can be used. +The commonground proto componant natively supports time traveling on all resources that are annotaded with the @Gedmo\Loggable, this is done by adding the ?validOn=[date] query to a request, date can either be a datetime or datedatime string. Any value supported by php's [strtotime()](https://www.php.net/manual/en/function.strtotime.php) is supported. Keep in mind that this returns the entity a as it was valid on that time or better put, the last changed version BEFORE that moment. To get a complete list of all changes on an item the [/audittrail](#Audittrail +) endpoint can be used. __solution__ In compliance with [schema.org](https://schema.org/validFrom) `geldigop`,`geldigvan` and `geldigtot` are implemented as `validOn`,`validFrom` and `validUntil`. And can be used a query parameters on collection operations/ @@ -234,7 +213,7 @@ In the [zaak-api](https://zaken-api.vng.cloud/api/v1/schema/#operation/zaak_list Translations ------- -We support translations trough the `Accept-Language` header (read the [docs](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language)), the fallback langouge for all messages is englisch +We support translations trough the `Accept-Language` header (read the [docs](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language)), the fallback language for all messages is English Errors ------- @@ -256,7 +235,7 @@ We support both comma and bracket notation on array's, but only document bracket Container Setup ------- - https://medium.com/shiphp/building-a-custom-nginx-docker-image-with-environment-variables-in-the-config-4a0c36c4a617 +https://medium.com/shiphp/building-a-custom-nginx-docker-image-with-environment-variables-in-the-config-4a0c36c4a617 Filtering @@ -268,7 +247,7 @@ __Regex Exact__ __Regex Contains__ __Like___ -The like filters is used to search for enities with the traditional sql LIKE operator. If pattern does not contain percent signs or underscores, then the pattern only represents the string itself; in that case LIKE acts like the equals operator. An underscore (_) in pattern stands for (matches) any single character; a percent sign (%) matches any sequence of zero or more characters. +The like filters is used to search for resources with the traditional sql LIKE operator. If pattern does not contain percent signs or underscores, then the pattern only represents the string itself; in that case LIKE acts like the equals operator. An underscore (_) in pattern stands for (matches) any single character; a percent sign (%) matches any sequence of zero or more characters. Some examples: @@ -276,6 +255,7 @@ Some examples: 'abc' LIKE 'a%' true 'abc' LIKE '_b_' true 'abc' LIKE 'c' false + LIKE pattern matching always covers the entire string. Therefore, if it's desired to match a sequence anywhere within a string, the pattern must start and end with a percent sign. To match a literal underscore or percent sign without matching other characters, the respective character in pattern must be preceded by a backlash. @@ -283,10 +263,32 @@ To match a literal underscore or percent sign without matching other characters, ## Kubernetes ### Loadbalancers -We no longer provide a load balancer per component, since this would require a ip per component. Draining ip's on mult component kubernetes clusters. In stead we make componentes available as an interner service +We no longer provide a load balancer per component, since this would require a IP address per component (and ipv 4 addresses are in short supply). Instead we make components available as an internal service. A central load balancer could then be used to provide several api’s in one ### server naming -A component is (speaking in kubernetes terms) a service that is available at +A component is (speaking in kubernetes terms) a service that is available at a name corresponding to its designation + +### Domain Build-up and routing +By convention the component assumes that you follow the common ground domain name build up, meaning {environment}.{component}.{rest of domain}. That means that only the first two url parts are used for routing. It is also assumed that when no environment is supplied the production environment should be offered E.g. a proper domain for the production API of the verzoeken registratie component would be prod.vrc.zaakonline.nl but it should also be reachable under vrc.zaakonline.nl. The proper location for the development environment should always be dev.vrc.zaakonlin.nl + +### Environments and namespacing +We assume that for that you want to run several environments for development purposes. We identify the following namespaces for support. +- prod (Production) +- acce (Acceptation) +- stag (Staging) +- test (Testing) +- dev (Development) + +Because we base the common ground infrastructure on kubernetes, and we want to keep a hard separation between environment we also assume that you are using your environment as a namespace + +Symfony library management gives us the option to define the libraries on a per environment base, you can find that definition in the [bundle config](api/config/bundles.php) + +Besides the API environments the component also ships with additional tools/environments but those are not meant to be deployed +- client (An react client frontend) +- admin (An read admin interface) + +On the local development docker deploy the client environment is used as default instead of the production version of the api. + ## Data types @@ -325,3 +327,4 @@ A component is (speaking in kubernetes terms) a service that is available at | string | iban | | | | | | | | | | | | + diff --git a/INSTALLATION.md b/INSTALLATION.md index 05bad3ed..96ea76d5 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -4,6 +4,7 @@ This document dives a little bit deeper into installing your component on a kube ## Setting up helm + ## Setting up tiller Create the tiller service account: @@ -18,39 +19,45 @@ $ kubectl create clusterrolebinding tiller --clusterrole cluster-admin --service Now we can run helm init, which installs Tiller on our cluster, along with some local housekeeping tasks such as downloading the stable repo details: ```CLI -$ helm init --service-account tiller --kubeconfig="api/helm/kubeconfig.yaml" +$ helm init --service-account tiller --kubeconfig="kubeconfig.yaml" ``` To verify that Tiller is running, list the pods in the kube-system namespace: ```CLI -$ kubectl get pods --namespace kube-system --kubeconfig="api/helm/kubeconfig.yaml" +$ kubectl get pods --namespace kube-system --kubeconfig="kubeconfig.yaml" ``` The Tiller pod name begins with the prefix tiller-deploy-. Now that we've installed both Helm components, we're ready to use helm to install our first application. + +## Setting up ingress +We need at least one nginx controller per kubernetes kluster, doh optionally we could set on up on a per namebase basis + +helm install stable/nginx-ingress --name loadbalancer --kubeconfig="kubeconfig.yaml" + ## Setting up Kubernetes Dashboard After we installed helm and tiller we can easily use both to install kubernetes dashboard ```CLI -$ helm install stable/kubernetes-dashboard --name dashboard --kubeconfig="api/helm/kubeconfig.yaml" --namespace="kube-system" +$ helm install stable/kubernetes-dashboard --name dashboard --kubeconfig="kubeconfig.yaml" --namespace="kube-system" ``` But before we can login to tiller we need a token, we can get one of those trough the secrets. Get yourself a secret list by running the following command ```CLI -$ kubectl -n kube-system get secret --kubeconfig="api/helm/kubeconfig.yaml" +$ kubectl -n kube-system get secret --kubeconfig="kubeconfig.yaml" ``` Because we just bound tiller to our admin account and use tiller (trough helm) to manage our code deployment it makes sense to use the tiller token, lets look at the tiller secret (it should look something like "tiller-token-XXXXX" and ask for the corresponding token. ```CLI -$ kubectl -n kube-system describe secrets tiller-token-xxxxx --kubeconfig="api/helm/kubeconfig.yaml" +$ kubectl -n kube-system describe secrets tiller-token-xxxxx --kubeconfig="kubeconfig.yaml" ``` This should return the token, copy it to somewhere save (just the token not the other returned information) and start up a dashboard connection ```CLI -$kubectl proxy --kubeconfig="api/helm/kubeconfig.yaml" +$kubectl proxy --kubeconfig="kubeconfig.yaml" ``` This should proxy our dashboard to helm making it available trough our favorite browser and a simple link diff --git a/TUTORIAL.md b/TUTORIAL.md index 74aad7ba..356121e3 100644 --- a/TUTORIAL.md +++ b/TUTORIAL.md @@ -8,11 +8,11 @@ What do you need for this tutorial? * Docker for desktop ## Before you begin -For the steps considering the generation of entities an example entity a availale, feel free to [take a look](https://github.com/ConductionNL/Proto-component-commonground/blob/master/api/src/Entity/ExampleEntity.php) at it if you have trouble figuring out the code. +For the steps considering the generation of resources (or entities as symfony calls them) an example resource a availale, feel free to [take a look](https://github.com/ConductionNL/Proto-component-commonground/blob/master/api/src/Entity/ExampleEntity.php) at it if you have trouble figuring out the code. ## Setting up your enviroment -You can install docker-desktop from [the docker website](). +You can install docker-desktop from [the docker website](https://hub.docker.com/editions/community/docker-ce-desktop-windows). ## Generating your component (repository/codebase) @@ -26,7 +26,7 @@ After that you should be redirected to your own brand new repository. We ran a fork of the base Common Ground component, that means that we copied the code of the original project into a new repository. By doing so we made sure we have all the necessities for our component, including security and compliance with international standards. ## Spinning up your component -Before we can spin up our component we must first get a local copy from our repository, we can either do this through the command line (example here) or use a Git client. +Before we can spin up our component we must first get a local copy from our repository, we can either do this through the command line or use a Git client. For this example we're going to use GitKraken but you can use any tool you like, feel free to skip this part if you are already familiar with setting up a local clone of your repository. @@ -35,7 +35,7 @@ Open gitkraken press "clone a repo" and fill in the form (select where on your l You can now navigate to the folder where you just installed your code, it should contain some folders and files and generally look like this. We will get into the files later, lets first spin up our component! Open a command window (example) and browse to the folder where you just stuffed your code, navigating in a command window is done by cd, so for our example we could type -cd c:\repos\common-ground\my-component (if you installed your code on a different disk then where the cmd window opens first type : for example D: and hit enter to go to that disk, D in this case). We are now in our folder, so let's go! Type docker-compose up and hit enter. From now on whenever we describe a command line command we will document it as follows: +cd c:\repos\common-ground\my-component (if you installed your code on a different disk then where the cmd window opens first type : for example D: and hit enter to go to that disk, D in this case). We are now in our folder, so let's go! Type docker-compose up and hit enter. From now on whenever we describe a command line command we will document it as follows (the $ isn't actually typed but represents your folder structure): ```CLI $ docker-compose up @@ -45,8 +45,9 @@ Your computer should now start up your local development environment. Don't worr Open your browser type http://localhost/ as address and hit enter, you should now see your common ground component up and running. -### trouble shooting +### Trouble shooting When spinning up components we make extensive use of the cashing of docker, and use volumes to represent server disks. When running in to unexpected trouble always remember to clear your local docker vm with the -a command (removing image cash) + ```CLI $ docker system prune -a ``` @@ -57,19 +58,19 @@ $ docker volume prune **What are we looking at?** The Common Ground base component provides a bit more than just a development interface, it also includes an example application and a backend that automatically hooks into your api. For now we're just going to focus on our api, but is good to read up on all the features of the Common Ground base component here. -## Adding your own objects -You can now access your api at http://localhost:8080/, as you can see it's pre-loaded with some example objects. Let's replace them with your own objects! +## Adding your own resources +You can now access your api at http://localhost:8080/, as you can see it's pre-loaded with some example resources. Let's replace them with your own resources! -First let's remove the objects currently in the api, we can do that by just removing the entities form our code base, navigate to the folder where you stored your code and open the folder api/src/Entity , you can find the example entities (our name for objects) there. Just delete all the php files in that folder. +First let's remove the resources currently in the api, we can do that by just removing the resources form our code base, navigate to the folder where you stored your code and open the folder api/src/Entity , you can find the example entities (the symfony name for resources) there. Just delete all the php files in that folder. -Next let's add our own entities, we can do this in two ways, we can do old fashioned coding, but we can also use the build in maker bundle of the proto component, to quickly generate our entities for us (without the fuss of actual coding). +Next let's add our own resources, we can do this in two ways, we can do old fashioned coding, but we can also use the build in maker bundle of the proto component, to quickly generate our entities for us (without the fuss of actual coding). Let's open a new command line window and navigate to our root folder, exactly like we did under "spinning up your component". And then lets fire up maker bundle (make sure that your component is still running in your other command window). We can do so by the following command: ```CLI $ docker-compose exec php bin/console make:entity ``` -We should now see a wizard that allows us to either make new entities, or add parameters to existing entities (by supplying the name of an existing entity). +We should now see a wizard that allows us to either make new entities, or add parameters to existing entities (by supplying the name of an existing resource). ## Keeping your repository up to date with the Conduction Common Ground component @@ -107,7 +108,7 @@ git merge upstream --allow-unrelated-histories Keep in mind that you wil need to make sure to stay up to date about changes on the Common Ground component repository. ## Renaming your component -Right now the name of your component is 'commonground' that's that's fine while running it locally or in its own kubernetes cluster but wil get you in when running it with other components when it without using a name space. So its good practice to name your component distinctly. But besides al of these practical reasons its of course also just cool to name your child before you unleash it on the unsuspecting common ground community. +Right now the name of your component is `commonground component` and its unique id `cg` that's that's fine while running it locally or in its own kubernetes cluster but wil get you in when running it with other components when it without using a name space. So its good practice to name your component distinctly. But besides al of these practical reasons its of course also just cool to name your child before you unleash it on the unsuspecting common ground community. Oke, so before we can nae the component we need to come up with a name. There are a couple of conventions here. First of the name should tell us what the component does, or is supposed to do with one or two words. So we would normally call an component about dogs the DogComponent and one about cats te CatComponent. The second convention is that we don't usually actually name our component 'component' but indicate its position in de common ground architecture. For that we have the following options: * Catalogus @@ -116,34 +117,11 @@ Oke, so before we can nae the component we need to come up with a name. There ar * Application * Tool -The we need to touch te following files +The actual name change is rather simple doh, just head over to the .env that contains all our config and change the apropriate variables * .env -* dockercompose.yaml -* api/.env -* api/helm/values.yaml -* api/docker/nginx/ - -## Adding more openapi documantation -```php -//... - /** - * @ApiProperty( - * attributes={ - * "openapi_context"={ - * "description" = "The name of a organisation", - * "type"="string", - * "format"="string", - * "example"="My Organisation" - * } - * } - * ) - */ - private $name; -//... -``` - -## Setting up security and access (also helps with serialization) +## Setting up security and access +We want to secure our resources in such a way that only users or applications with propper right can acces and update properties. ```PHP // src/Entity/Organisation.php @@ -169,7 +147,7 @@ class Organisation ``` ## Using validation -Right now we are just accepting data and passing them on to the database, and in a mock or poc context this is fine. Most of the calls will end up being get requests anyway. But in case that we actually want our clients to make post to the api it would be wise to add some validation to the fields we are recieving. Luckely for us the component comes pre packed with a valdiation tool that we can configure from our entity through annotion. If we for example want to make a field required we could do so as follows: +Right now we are just accepting data and passing them on to the database, and in a mock or poc context this is fine. Most of the calls will end up being get requests anyway. But in case that we actually want our clients to make post to the api it would be wise to add some validation to the fields we are recieving. Luckely for us the component comes pre packed with a valdiation tool that we can configure from our resources through annotion. If we for example want to make a field required we could do so as follows: ```PHP // src/Entity/Organisation.php @@ -193,10 +171,10 @@ class Organisation Keep in mind that we need to add the assert annotation to our class dependencies under 'use'. -More inforation on using validation can be found at the [symfony website](https://symfony.com/doc/current/validation.html), but it is als worth nothing that tis component comes pre packed with some typical NL validators like BSN. You can find those [here](). +More information on using validation can be found at the [symfony website](https://symfony.com/doc/current/validation.html), but it is als worth nothing that tis component comes pre packed with some typical NL validators like BSN. You can find those [here](). ## Using UUID -As default doctrine uses auto increment integers as identifiers (1,2, etc). For modern web applications we however prefer the use of UUID's. (e.g. e2984465-190a-4562-829e-a8cca81aa35d). Why? Wel for one it is more secure integer id's are easily guessable and make it possible to "ask" endpoint about objects that you should not know about. But UUID's also have a benefit in future proofing the application. If we in the future want to merge a table with another table (for example because two organisations using a component perform a merger) then we would have to reassign al id's and relations if we where using int based id's (both tables would have a row 1,2 etc) with UUID's however the change of doubles range somewhere in the billions. Meaning that it is likely that we only need to either reidentify only a handful of rows or more likely none at al! Turning our entire migration into a copy paste action. +As default doctrine uses auto increment integers as identifiers (1,2, etc). For modern web applications we however prefer the use of UUID's. (e.g. e2984465-190a-4562-829e-a8cca81aa35d). Why? Wel for one it is more secure integer id's are easily guessable and make it possible to "ask" endpoint about resources that you should not know about. But UUID's also have a benefit in future proofing the application. If we in the future want to merge a table with another table (for example because two organisations using a component perform a merger) then we would have to reassign al id's and relations if we where using int based id's (both tables would have a row 1,2 etc) with UUID's however the change of doubles range somewhere in the billions. Meaning that it is likely that we only need to either reidentify only a handful of rows or more likely none at al! Turning our entire migration into a copy paste action. The proto component supports Ramsey's uuid objects strategy out of the box, so to use UUID's as identifier simply we need to add the ApiProperty as a dependency @@ -222,20 +200,9 @@ with ```PHP //... - /** - * @var \Ramsey\Uuid\UuidInterface - * - * @ApiProperty( - * identifier=true, - * attributes={ - * "openapi_context"={ - * "description" = "The UUID identifier of this object", - * "type"="string", - * "format"="uuid", - * "example"="e2984465-190a-4562-829e-a8cca81aa35d" - * } - * } - * ) + /** + * @var \Ramsey\Uuid\UuidInterface The UUID identifier of this resource + * @example e2984465-190a-4562-829e-a8cca81aa35d * * @Groups({"read"}) * @ORM\Id @@ -247,7 +214,7 @@ with //.. ``` -and remove the integer on the getter turning this: +and remove the : ?integer on the getter turning this: ```PHP //... @@ -272,7 +239,7 @@ into this and you're all done ### Trouble shooting -If you have already spun your component including your new entity your going to run into some trouble because doctrine is going to try changing your primary key column (id) from an integer to string (tables tend not to like that). In that case its best to just drop your database and reinstall it using the following commands: +If you have already spun your component including your new resource your going to run into some trouble because doctrine is going to try changing your primary key column (id) from an integer to string (tables tend not to like that). In that case its best to just drop your database and reinstall it using the following commands: ```CLI $ bin/console doctrine:schema:drop @@ -281,9 +248,9 @@ $ bin/console doctrine:schema:update --force ## Advanced data sets -Oke lets make it complex, until now we have just added some simple entities to our component, but what if we want to attaches one entity to another? Fortunately our build in database engine support rather complex scenarios called associations. So let [take a look](https://www.doctrine-project.org/projects/doctrine-orm/en/2.6/reference/association-mapping.html) at that. +Oke lets make it complex, until now we have just added some simple entities to our component, but what if we want to attaches one resource to another? Fortunately our build in database engine support rather complex scenarios called associations. So let [take a look](https://www.doctrine-project.org/projects/doctrine-orm/en/2.6/reference/association-mapping.html) at that. -Baffled? Wel its rather complex. But remember that Make:entity command that we used earlier? That actually accepts relations as a data type. Or to but it simply instead of using the default 'string' we could just type "ManyToOne" and it will just fire up some questions that will help it determine how you want your relations to be. +Baffled? Wel its rather complex. But remember that make:entity command that we used earlier? That actually accepts relations as a data type. Or to but it simply instead of using the default 'string' we could just type "ManyToOne" and it will just fire up some questions that will help it determine how you want your relations to be. ### Trouble shooting @@ -315,7 +282,7 @@ We can now prevent circular references by setting a max depth on the properties ```PHP //... /** - * @var ArrayCollection $stuffs Some stuff that is attached to this example object + * @var ArrayCollection $stuffs Some stuff that is attached to this example resource * * @MaxDepth(1) * @Groups({"read","write"}) @@ -326,7 +293,7 @@ We can now prevent circular references by setting a max depth on the properties ``` ## Data fixtures -For testing cases it can be useful to use data fixtures a predefined set of data that fills the database of your component at startup. Since we use php classes to describe our objects creating fixtures is easy (you can find an example in your project folder at api/src/DataFixtures). We simply go trough some classes assign values and persist them to the database. Once we have written our fixtures we can use a single command to load them +For testing cases it can be useful to use data fixtures a predefined set of data that fills the database of your component at startup. Since we use php classes to describe our resources creating fixtures is easy (you can find an example in your project folder at api/src/DataFixtures). We simply go trough some classes assign values and persist them to the database. Once we have written our fixtures we can use a single command to load them ```CLI $ bin/console doctrine:fixtures:load --env=dev @@ -350,42 +317,39 @@ When using Github. To set up a webhook, go to the settings page of your reposito Now every time you update your repository the commonground dev page will alerted, rescan your repository and do al the appropriate platform actions. It just as easy as that. -Automated Testing and Deployment (continues integration) +Continues integration ------- -The following bit of the tutorial requires two additional accounts -- [https://hub.docker.com/](https://hub.docker.com/) (You might already have this for docker for desktop) -- [https://travis-ci.org](https://travis-ci.org) (You can use you github account) - -The proto component ships with a pre-fab continues integration script based on travis. What does this mean you ask? Continuous integration (or CI for short) is an optimized and automated way for your code to become part of your projects. In the case of your commonground component that means that we will automatically validate new code commits or pushes and (if everything checks out) build that code and deploy the containers thereof to docker hub. Making is possible to update al the environments that use those components. - -Okay, that's nice, but how do we do that? Actually it is very simple. First of all make sure you have a docker account, log into [docker hub](https://hub.docker.com/) and have a look around. We don't need to create anything just yet, but it is nice to get a feeling of the place. As you can see docker hub also uses repositories etc. So that recognizable. +> The following bit of the tutorial requires an additional account +> - [https://hub.docker.com/](https://hub.docker.com/) (You might already have this for docker for desktop) -Next we need to prepare our github repository that holds our code. For the travis script to work as intended we need to create a couple of branches(if we don't have those already) open up your git interface and create a branch called 'development' and a branch called 'staging'. Don't forget to push the branches so that they are present on github (and not just on your local machine). +The proto component ships with a pre-fab continues integration script based on github action (there is also a travis script in here if you want it). What does this mean you ask? Continuous integration (or CI for short) is an optimized and automated way for your code to become part of your projects. In the case of your commonground component that means that we will automatically validate new code commits or pushes and (if everything checks out) build that code and deploy the containers thereof to docker hub. Making is possible to update al the environments that use those components. Whats even better is that we check your code for known security issues, so whenever a dependency or libary has a security issue you will be notified to take action. -Oke just one more place to go and that is travis, head over to [https://travis-ci.org](https://travis-ci.org) and login with your github account. If everything is alright you should see your repository there. Activate it by pressing 'activate repository' and then go to 'More options' -> 'Settings' and scroll down to environment variables. Here we can present travis wit the variables that it need to execute our build script. Lets first set the common variables that we need for all our branches: `DOCKER_PASSWORD` your docker password,`DOCKER_REGISTRY` docker.io/[your username] ,`DOCKER_USERNAME` your docker user name. This will be used by travis to push the completed containers into docker hub. Next we need to specify a couple of variables that are branch specific. Or to be more exact, set the same variable `APP_ENV` with different values for different branches. It needs to be 'staging'->stag,'master'->prod,'development'->dev. +Okay, that's nice, but how do we do that? Actually it is very simple. You do nothing. The scripts are already enabled by default. Just go to the actions tab of your github repository to see the results whenever you push code. -And all done! Head over back to the code on your computer and make a small change. Then commit push that change into github. Travis should automatically pick op your change and start a build. +There is however a bit of extra here that you can do and that is to insert your docker hub credentials into the repository. You can do that under the settings->secrets tab of yout repoistory by setting a `DOCKERHUB_USERNAME` and `DOCKERHUB_PASSWORD` secret containing (you might have guesed it) your dockerhub username and secret. And all done! Head over back to the code on your computer and make a small change. Then commit push that change into github. Wait for the action to complete and head over to your docker hub repository page. You should find your build containers ready for you. +Continues deployment +------- +> The following bit of the tutorial requires an additional account +> - [https://www.digitalocean.com/](https://www.digitalocean.com/) -### Unit / Behat - -TODO +Actually the repository goes a bit further then just getting your containers ready to deploy, it can acctually deploy them for you! Again all the code is already there. The only thing that you need to do is add a kubeconfig file. You can get a kubeconfig file from a running kubernetes clusters, it provides your repository with both the credentials and endpoints it needs to deploy the application. How you get a Kubeconfig file difers a bit from provider to provider. But you can get more information on that here -### Postman -TODO +- [Digitalocean](https://www.digitalocean.com/docs/kubernetes/how-to/connect-to-cluster/) +- [Google Cloud](https://cloud.google.com/sdk/gcloud/reference/container/clusters/get-credentials) +- [Amazone AWS](https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html) -### Trouble shooting -Please make sure that your github repository is set to public, and keep in mind that a complex travis build (and certainly one that includes a pushing of containers can take up to 20 minutes). +Afther you have abtained a kuneconfig you need to save it to your repository as a secret (NEVER COMMIT A KUBECONFIG FILE), use the secret `KUBECONFIG` to save your cubeconfig file. Now simply commit and push your code to your repository and presto! You have a working common-ground component online. Documentation and dockblocks ------- -TODO +You want both your redoc documentation and your code to be readable and reausable to other developers. To this effect we use docblok annotation. You can read more about that [here](https://docs.phpdoc.org/references/phpdoc/basic-syntax.html) but the basic is this, we supply each class and propery with a docblock contained within /\* \* / characters. At the very least we want to describe our properties, the expected results and example data (see the example under [audittrail](#audittrail) Audittrail ------- As you might expect the proto-component ships with a neat function for generating audit trails, that basically exist of three parts. -First we need to activate logging on the entities that we want logged (for obvious security reasons we don't log entity changes by default) to do that by adding the `@Gedmo\Loggable` annotation to our php class, which should then look something like: +First we need to activate logging on the entities that we want logged (for obvious security reasons we don't log resource changes by default) to do that by adding the `@Gedmo\Loggable` annotation to our php class, which should then look something like: ```PHP //... @@ -440,17 +404,3 @@ class ExampleEntity ``` And now we have a fully nl api strategy integrated audit trail! - - -Setting up automated deployment (continues delivery) -------- -TODO - -## Commonground specific data types -TODO - -### incompleteDate - - -### underInvestigation - diff --git a/api/.env b/api/.env index a2472076..235c628f 100644 --- a/api/.env +++ b/api/.env @@ -27,8 +27,8 @@ CONDUCTION_COMMONGROUND_BAG_APIKEY=!ChangeMe! #APP_NAME='pc' APP_DESCRIPTION='Naast deze JSON rest API is er ook een [graphql](/graphql) interface beschikbaar.' APP_SECRET=!ChangeMe! -TRUSTED_PROXIES=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 -TRUSTED_HOSTS='^(.+\.)?localhost|api$' +#TRUSTED_PROXIES=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 +#TRUSTED_HOSTS='^(.+\.)?localhost|api$' ###< symfony/framework-bundle ### ###> symfony/framework-bundle ### diff --git a/api/config/packages/twig.yaml b/api/config/packages/twig.yaml index 1a687ce2..8284abd9 100644 --- a/api/config/packages/twig.yaml +++ b/api/config/packages/twig.yaml @@ -14,6 +14,7 @@ twig: app_version: '%env(APP_VERSION)%' app_env: '%env(APP_ENV)%' app_debug: '%env(APP_DEBUG)%' + app_domain: '%env(APP_DOMAIN)%' app_demo: '%env(APP_DEMO)%' app_repro: '%env(APP_REPRO)%' app_description: '%env(APP_DESCRIPTION)%' @@ -21,7 +22,10 @@ twig: app_auth: '%env(AUTH_ENABLED)%' app_audittrail: '%env(AUDITTRAIL_ENABLED)%' app_notification: '%env(NOTIFICATION_ENABLED)%' - app_health: '%env(HEALTH_ENABLED)%' + app_health: '%env(HEALTH_ENABLED)%' + + trusted_hosts: '%env(TRUSTED_HOSTS)%' + trusted_proxies: '%env(TRUSTED_PROXIES)%' nlx_outway: '%env(NLX_OUTWAY)%' nlx_inway: '%env(NLX_INWAY)%' diff --git a/api/helm/templates/configmap.yaml b/api/helm/templates/configmap.yaml index 55279410..d3b97459 100644 --- a/api/helm/templates/configmap.yaml +++ b/api/helm/templates/configmap.yaml @@ -16,6 +16,7 @@ data: app-version: {{ .Values.settings.version | quote }} app-repro: {{ .Values.settings.repro | quote }} app-demo: {{ .Values.settings.demo | quote }} + app-domain: {{ .Values.settings.domain | quote }} app-description: {{ .Values.settings.description | quote }} app-auth: {{ .Values.settings.authorisationEnabled | quote }} diff --git a/api/helm/templates/ingress.yaml b/api/helm/templates/ingress.yaml index 4437f785..1c549535 100644 --- a/api/helm/templates/ingress.yaml +++ b/api/helm/templates/ingress.yaml @@ -1,7 +1,7 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: - name: {{ template "fullname" . }} + name: {{ include "name" . }}-{{ .Values.settings.env }}-ingress labels: app.kubernetes.io/name: {{ include "name" . }}-ingress app.kubernetes.io/part-of: {{ include "name" . }} @@ -24,12 +24,39 @@ spec: {{- end }} {{- end }} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - #host: {{ .Values.settings.name }}.{{ .Values.settings.env }}.{{ .Values.settings.domain }} http: paths: - - path: /* - backend: - serviceName: {{ .serviceName }} - servicePort: {{ .servicePort | default 80 }} - {{- end }} + - backend: + serviceName: {{ include "name" . }} + servicePort: 80 + - host: {{ .Values.settings.name }}.{{ .Values.settings.env }}.{{ .Values.settings.domain }} + http: + paths: + - backend: + serviceName: {{ include "name" . }} + servicePort: 80 + - host: {{ .Values.settings.name }}.{{ .Values.settings.env }}.huwelijksplanner.online + http: + paths: + - backend: + serviceName: {{ include "name" . }} + servicePort: 80 + - host: {{ .Values.settings.name }}.{{ .Values.settings.env }}.trouwplanner.online + http: + paths: + - backend: + serviceName: {{ include "name" . }} + servicePort: 80 + - host: {{ .Values.settings.name }}.{{ .Values.settings.env }}.common-ground.nl + http: + paths: + - backend: + serviceName: {{ include "name" . }} + servicePort: 80 + - host: {{ .Values.settings.name }}.{{ .Values.settings.env }}.larping.eu + http: + paths: + - backend: + serviceName: {{ include "name" . }} + servicePort: 80 diff --git a/api/helm/templates/php-deployment.yaml b/api/helm/templates/php-deployment.yaml index 1b03232e..3f09ef5d 100644 --- a/api/helm/templates/php-deployment.yaml +++ b/api/helm/templates/php-deployment.yaml @@ -77,6 +77,11 @@ spec: configMapKeyRef: name: {{ template "fullname" . }} key: app-demo + - name: APP_DOMAIN + valueFrom: + configMapKeyRef: + name: {{ template "fullname" . }} + key: app-domain - name: APP_REPRO valueFrom: configMapKeyRef: diff --git a/api/helm/values.yaml b/api/helm/values.yaml index 6bf6d3b2..3f474c4a 100644 --- a/api/helm/values.yaml +++ b/api/helm/values.yaml @@ -11,11 +11,12 @@ settings: description: ''Naast deze JSON rest API is er ook een [graphql](/graphql) interface beschikbaar.'' repro: https://github.com/ConductionNL/Proto-component-commonground' demo: pc.zaakonline.nl + domain: conduction.nl env: dev debug: 1 replicaCount: 1 corsAllowOrigin: ['*'] - trustedHosts: '^(.+\.)?common-ground\.dev$|^(.+\.)?zaakonline\.nl$|^(.+\.)?conduction\.nl$|^example\.com$|^(.+\.)?178.128.142.152$|178.128.142.152|localhost' + trustedHosts: '^(.+\.)?conduction\.nl$|^(.+\.)?huwelijksplanner\.online$|^(.+\.)?larping\.eu$|^(.+\.)?common-ground\.nl$|^(.+\.)?trouwplanner\.online$|^(.+\.)?zaakonline\.nl$|localhost' pullPolicy: Always # You will need these proxies on kubernetes trustedProxies: @@ -94,13 +95,6 @@ ingress: # - hosts: # - example.com # - mercure.example.com - hosts: - api: - host: example.com - serviceName: varnish - mercure: - host: mercure.example.com - serviceName: mercure resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/api/src/Swagger/SwaggerDecorator.php b/api/src/Swagger/SwaggerDecorator.php index 045ef50c..28b4d3e5 100644 --- a/api/src/Swagger/SwaggerDecorator.php +++ b/api/src/Swagger/SwaggerDecorator.php @@ -127,6 +127,12 @@ public function normalize($object, $format = null, array $context = []) // Oke dit is echt but lelijk $schemas = (array) $docs['definitions']; foreach ($schemas as $schemaName => $schema) { + + // We can only merge if we actually have content + if (!in_array($schemaName, $additionalDocs)) { + continue; + } + $additionalDocs[$schemaName] = array_merge((array) $schema, $additionalDocs[$schemaName]); $properties = (array) $schema['properties']; @@ -407,6 +413,7 @@ private function getAdditionalEntityDocs($entity) $docblock = $factory->create($property->getDocComment()); $tags = $docblock->getTags(); $atributes = []; + $groups = []; foreach ($tags as $tag) { $name = $tag->getName(); diff --git a/api/templates/helm/Values.yaml.twig b/api/templates/helm/Values.yaml.twig index 33aebbd8..8b6ef17f 100644 --- a/api/templates/helm/Values.yaml.twig +++ b/api/templates/helm/Values.yaml.twig @@ -10,12 +10,13 @@ settings: version: {{ app_version }} description: '{{ app_description }}' repro: {{ app_repro }}' + domain: {{ app_domain }} demo: {{ app_demo }} env: {{ app_env }} debug: {{ app_debug }} replicaCount: 1 corsAllowOrigin: ['*'] - trustedHosts: '^(.+\.)?common-ground\.dev$|^(.+\.)?zaakonline\.nl$|^(.+\.)?conduction\.nl$|^example\.com$|^(.+\.)?178.128.142.152$|178.128.142.152|localhost' + trustedHosts: {{ trusted_hosts }} pullPolicy: Always # You will need these proxies on kubernetes trustedProxies: diff --git a/docker-compose.yml b/docker-compose.yml index 43498f17..51c9655a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,9 +36,12 @@ services: - APP_VERSION=${APP_VERSION} - APP_NAME=${APP_NAME} - APP_TITLE=${APP_TITLE} + - APP_DOMAIN=${APP_DOMAIN} - APP_DEMO=${APP_DEMO} - APP_REPRO=${APP_REPRO} - APP_DESCRIPTION=${APP_DESCRIPTION} + - TRUSTED_PROXIES=${TRUSTED_PROXIES} + - TRUSTED_HOSTS=${TRUSTED_HOSTS} - AUTH_ENABLED=${AUTH_ENABLED} - AUDITTRAIL_ENABLED=${AUDITTRAIL_ENABLED} - NOTIFICATION_ENABLED=${NOTIFICATION_ENABLED}