From 5392673879d43820a194f187449127a201d175b0 Mon Sep 17 00:00:00 2001 From: Jo Humphrey <31373245+jamdelion@users.noreply.github.com> Date: Mon, 2 Dec 2024 13:50:58 +0000 Subject: [PATCH] docs: How to debug failed submissions (#4018) --- .../how-to-setup-aws-s3-submissions.md | 0 .../how-to-setup-aws-sso-credentials.md | 0 doc/how-to/how-to-debug-failed-submissions.md | 99 ++++++++++++++++++ doc/how-to/how-to-resubmit-an-application.md | 12 ++- .../how-to-add-allow-list-variables.md | 0 .../how-to-grant-metabase-permissions.md | 0 ...etup-metabase-for-a-new-team-or-service.md | 0 .../default_values_filtered.png | Bin .../setup-metabase/duplicate_a_dashboard.png | Bin .../setup-metabase/edit_flow_in_hasura.png | Bin .../images/setup-metabase/enable_sharing.png | Bin .../images/setup-metabase/new_collection.png | Bin .../only_duplicate_dashboard.png | Bin .../service_slug_default_value.png | Bin .../images/setup-metabase/share_with_team.png | Bin .../team_slug_default_value.png | Bin .../images/setup-metabase/templates.png | Bin .../setup-metabase/update_analytics_link.png | Bin .../{ => secrets}/how-to-add-a-secret.md | 0 .../{ => secrets}/how-to-add-a-team-secret.md | 0 .../{ => secrets}/how-to-generate-a-secret.md | 0 21 files changed, 106 insertions(+), 5 deletions(-) rename doc/how-to/{ => aws}/how-to-setup-aws-s3-submissions.md (100%) rename doc/how-to/{ => aws}/how-to-setup-aws-sso-credentials.md (100%) create mode 100644 doc/how-to/how-to-debug-failed-submissions.md rename doc/how-to/{ => metabase}/how-to-add-allow-list-variables.md (100%) rename doc/how-to/{ => metabase}/how-to-grant-metabase-permissions.md (100%) rename doc/how-to/{ => metabase}/how-to-setup-metabase-for-a-new-team-or-service.md (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/default_values_filtered.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/duplicate_a_dashboard.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/edit_flow_in_hasura.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/enable_sharing.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/new_collection.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/only_duplicate_dashboard.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/service_slug_default_value.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/share_with_team.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/team_slug_default_value.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/templates.png (100%) rename doc/how-to/{ => metabase}/images/setup-metabase/update_analytics_link.png (100%) rename doc/how-to/{ => secrets}/how-to-add-a-secret.md (100%) rename doc/how-to/{ => secrets}/how-to-add-a-team-secret.md (100%) rename doc/how-to/{ => secrets}/how-to-generate-a-secret.md (100%) diff --git a/doc/how-to/how-to-setup-aws-s3-submissions.md b/doc/how-to/aws/how-to-setup-aws-s3-submissions.md similarity index 100% rename from doc/how-to/how-to-setup-aws-s3-submissions.md rename to doc/how-to/aws/how-to-setup-aws-s3-submissions.md diff --git a/doc/how-to/how-to-setup-aws-sso-credentials.md b/doc/how-to/aws/how-to-setup-aws-sso-credentials.md similarity index 100% rename from doc/how-to/how-to-setup-aws-sso-credentials.md rename to doc/how-to/aws/how-to-setup-aws-sso-credentials.md diff --git a/doc/how-to/how-to-debug-failed-submissions.md b/doc/how-to/how-to-debug-failed-submissions.md new file mode 100644 index 0000000000..6a6da64d33 --- /dev/null +++ b/doc/how-to/how-to-debug-failed-submissions.md @@ -0,0 +1,99 @@ +# How to debug failed submissions + +This guide will explain how to troubleshoot failed application submissions. + +If a live application submission fails it is important to investigate this urgently, and aim for same-day re-submission if possible. Luckily this doesn't happen very often! + +More information on how to re-submit applications can be found in [./how-to-resubmit-an-application.md](./how-to-resubmit-an-application.md). + +## Finding out that a submission has failed + +### Via `#planx-errors` + +Normally we find out that a submission has failed through a notification in the `#planx-errors` Slack channel, and these are usually easy to spot as they are longer than the normal errors. + +They also start with `Error: Sending to failed`. + +E.g. + +```json +> New production error in planx (error severity) +> +Error: Sending to BOPS v2 failed (southwark - a-5e5510n-1d-a-535510n1d): Error: Invalid DigitalPlanning payload for session a-5e5510n-1d-a-535510n1d. Errors: [ { "instancePath": "/data/applicant/ownership/owners/0", "schemaPath": "#/required", "keyword": "required", "params": { "missingProperty": "noticeGiven" }, "message": "must have required property 'noticeGiven'" }, { "instancePath": "/data/applicant/ownership/owners/0", "schemaPath": "#/required", "keyword": "required", "params": { "missingProperty": "noNoticeReason" }, "message": "must have required property 'noNoticeReason'" }, { "instancePath": "/data/applicant/ownership/ow… +> Show more +> Occurred in file:///api/dist/modules/send/bops/bops.js:98 + +``` + +### Via `#planx-notifications` + +There are normally three notifications per Lawful Development Certificate (LDC) submission in the `#planx-notifications` Slack channel, so a tell-tale sign of an error is if there are fewer messages. + +For example, along with the production error above, for this submission there were only these two messages (the BOPS submission was missing): + +``` +6:02PM New GOV Pay payment cj14av1r8hd8vdqp5e2tn5gto1 with status success [southwark] +6:03PM New Uniform submission a-5e5510n-1d-a-535510n1d [Southwark] +``` + +When we successfully re-submitted the application, we got this third notification: + +``` +11:06AM New BOPS submission 24-00242-LDCP [https://southwark.bops.services/api/v2/planning_applications] +``` + +### Feedback from users + +Ideally we never want to get in the situation where an end-user notices the error before we do. But we may be informed about a failed submission via Slack channels shared with Local Planning Authorities (LPAs), such as the Open Digital Planning (ODP) Slack workspace. + +## Investigating the error + +We have several admin API endpoints that exist to help debug submission issues. These can be viewed in the [Swagger docs](https://api.editor.planx.uk/docs/#/admin). + +If it is not obvious from the error notification, you can follow these steps to investigate further: + +#### Inspect the whole error message + +1. Get the `sessionId` from the error notification. +2. In a browser, hit `api.editor.planx.uk/admin/session/{sessions-id}/digital-planning-application`. This should output the full error message. + +#### Inspect the erroring payload +3. To get the schema payload that caused this error, add a `skipValidation` query param to the `digital-planning-application` URL, i.e.`?skipValidation=true` + +#### Compare the payload with the schema +1. There are useful online tools which can help find the validation issue, such as `json-schema.app`. +2. Load the current schema URL (e.g. `https://theopensystemslab.github.io/digital-planning-data-schemas/v0.7.1/schemas/application.json` ), and you can compare the erroring payload directly with the schema via the window on the right in the same tool. + +> [!TIP] +> The schema URL can be found at the bottom of the payload: +> +> `{ ..."schema": "https://theopensystemslab.github.io/digital-planning-data-schemas/v0.7.1/schemas/application.json" }` + +#### Read the code +1. The flow passport is mapped to the schema in this file: https://github.com/theopensystemslab/planx-core/blob/005ef823e68eaf9f7d7d5a2c793a6a729e0b6475/src/export/digitalPlanning/model.ts + +#### Inspect the Hasura record for the submission +1. Use the admin `/summary` endpoint. This will output the hasura record for the submission, including the entire passport. + +#### Go through the flow that the submission came from +1. Trace the journey that the user took based on their passport variables, and find out if/where there is a mismatch between content and schema. + +> [!TIP] +> It might be useful to toggle the data fields for the flow to help match the passport answers with the questions. + +> [!WARNING] +> Note that the published flow that the applicant used might be different from the current version of the editor flow! + +## Fixing the payload + +If the solution involves fixing the submission payload before resubmitting, you can do this as follows: + +1. Go to the production hasura console and filter the `lowcal_sessions` table with the `session_id`. +2. Make a copy of the contents of the `data` field (it is a good idea to store this temporarily in your code editor in case multiple changes are needed). +3. Fix the payload (e.g. this might mean inserting a new key-value pair in the JSON) and validate the payload with a tool like [jsonlint.com](jsonlint.com). +4. Replace the data field in Hasura with the corrected and validated payload. +5. Save the data, then rerun the admin endpoints from earlier (without the `skipValidation` param) to check that there are no longer any errors. +6. If all is good, resubmit the application - follow the instructions in [./how-to-resubmit-an-application.md](./how-to-resubmit-an-application.md). + +> [!CAUTION] +> Production data should only be edited when pairing with another OSL developer! diff --git a/doc/how-to/how-to-resubmit-an-application.md b/doc/how-to/how-to-resubmit-an-application.md index 033cce136f..95ba599d7b 100644 --- a/doc/how-to/how-to-resubmit-an-application.md +++ b/doc/how-to/how-to-resubmit-an-application.md @@ -27,13 +27,15 @@ In the near future, this process should be replaced by a re-submission API which * `bops_application.response.message` should be changed from `"Application created"` to `"FAILED"` 6. In Insomnia (or HTTP client of choice), prepare to re-submit the payload - - * API Endpoint - `https://api.editor.planx.uk/${destination}/${localAuthority}` - * For example, https://api.editor.planx.uk/uniform/aylesbury-vale + * API Endpoint - `https://api.editor.planx.uk/create-send-events/${sessionId}` + * For example, https://api.editor.planx.uk/create-send-events/a535510n-1d-a535510n-1d * HTTP Method - POST * Headers - `{ "Authorization": ${HASURA_PLANX_API_KEY} }` * This secret can be found via Pulumi or your `.env` file + +This can also be done directly from the [Swagger docs](https://api.editor.planx.uk/docs/). -> ⚠️ *Uniform instances do not neatly map to local authorities. Please take care to ensure that the value for `localAuthority` is taken from `uniform_applications.destination` when re-submitting to Uniform* +> ⚠️ *Uniform instances do not neatly map to local authorities. Please take care to ensure that the value for `localAuthority` in the payload is taken from `uniform_applications.destination` when re-submitting to Uniform* 7. Prepare the payload copied from Step 3. Ensure the structure is correct, see example - @@ -47,9 +49,9 @@ In the near future, this process should be replaced by a re-submission API which 8. Send! ✉️ -9. Check response message from BoPS / Uniform to confirm success or failure. Rectify any issues if possible (e.g. malfored payload) +9. Check response message from BoPS / Uniform to confirm success or failure (e.g. you can check the submissions log in the PlanX Editor). Rectify any issues if possible (e.g. malformed payload) -10. Notify partners on `#planx-notifications` channel of the re-submissions +10. Notify partners on `#planx-notifications` channel of the re-submissions. It is useful to link missing submissions to their counterpart payment logs, if applicable (reply in thread). ## Process if the application has been paid for, but there are no submission attempts recorded diff --git a/doc/how-to/how-to-add-allow-list-variables.md b/doc/how-to/metabase/how-to-add-allow-list-variables.md similarity index 100% rename from doc/how-to/how-to-add-allow-list-variables.md rename to doc/how-to/metabase/how-to-add-allow-list-variables.md diff --git a/doc/how-to/how-to-grant-metabase-permissions.md b/doc/how-to/metabase/how-to-grant-metabase-permissions.md similarity index 100% rename from doc/how-to/how-to-grant-metabase-permissions.md rename to doc/how-to/metabase/how-to-grant-metabase-permissions.md diff --git a/doc/how-to/how-to-setup-metabase-for-a-new-team-or-service.md b/doc/how-to/metabase/how-to-setup-metabase-for-a-new-team-or-service.md similarity index 100% rename from doc/how-to/how-to-setup-metabase-for-a-new-team-or-service.md rename to doc/how-to/metabase/how-to-setup-metabase-for-a-new-team-or-service.md diff --git a/doc/how-to/images/setup-metabase/default_values_filtered.png b/doc/how-to/metabase/images/setup-metabase/default_values_filtered.png similarity index 100% rename from doc/how-to/images/setup-metabase/default_values_filtered.png rename to doc/how-to/metabase/images/setup-metabase/default_values_filtered.png diff --git a/doc/how-to/images/setup-metabase/duplicate_a_dashboard.png b/doc/how-to/metabase/images/setup-metabase/duplicate_a_dashboard.png similarity index 100% rename from doc/how-to/images/setup-metabase/duplicate_a_dashboard.png rename to doc/how-to/metabase/images/setup-metabase/duplicate_a_dashboard.png diff --git a/doc/how-to/images/setup-metabase/edit_flow_in_hasura.png b/doc/how-to/metabase/images/setup-metabase/edit_flow_in_hasura.png similarity index 100% rename from doc/how-to/images/setup-metabase/edit_flow_in_hasura.png rename to doc/how-to/metabase/images/setup-metabase/edit_flow_in_hasura.png diff --git a/doc/how-to/images/setup-metabase/enable_sharing.png b/doc/how-to/metabase/images/setup-metabase/enable_sharing.png similarity index 100% rename from doc/how-to/images/setup-metabase/enable_sharing.png rename to doc/how-to/metabase/images/setup-metabase/enable_sharing.png diff --git a/doc/how-to/images/setup-metabase/new_collection.png b/doc/how-to/metabase/images/setup-metabase/new_collection.png similarity index 100% rename from doc/how-to/images/setup-metabase/new_collection.png rename to doc/how-to/metabase/images/setup-metabase/new_collection.png diff --git a/doc/how-to/images/setup-metabase/only_duplicate_dashboard.png b/doc/how-to/metabase/images/setup-metabase/only_duplicate_dashboard.png similarity index 100% rename from doc/how-to/images/setup-metabase/only_duplicate_dashboard.png rename to doc/how-to/metabase/images/setup-metabase/only_duplicate_dashboard.png diff --git a/doc/how-to/images/setup-metabase/service_slug_default_value.png b/doc/how-to/metabase/images/setup-metabase/service_slug_default_value.png similarity index 100% rename from doc/how-to/images/setup-metabase/service_slug_default_value.png rename to doc/how-to/metabase/images/setup-metabase/service_slug_default_value.png diff --git a/doc/how-to/images/setup-metabase/share_with_team.png b/doc/how-to/metabase/images/setup-metabase/share_with_team.png similarity index 100% rename from doc/how-to/images/setup-metabase/share_with_team.png rename to doc/how-to/metabase/images/setup-metabase/share_with_team.png diff --git a/doc/how-to/images/setup-metabase/team_slug_default_value.png b/doc/how-to/metabase/images/setup-metabase/team_slug_default_value.png similarity index 100% rename from doc/how-to/images/setup-metabase/team_slug_default_value.png rename to doc/how-to/metabase/images/setup-metabase/team_slug_default_value.png diff --git a/doc/how-to/images/setup-metabase/templates.png b/doc/how-to/metabase/images/setup-metabase/templates.png similarity index 100% rename from doc/how-to/images/setup-metabase/templates.png rename to doc/how-to/metabase/images/setup-metabase/templates.png diff --git a/doc/how-to/images/setup-metabase/update_analytics_link.png b/doc/how-to/metabase/images/setup-metabase/update_analytics_link.png similarity index 100% rename from doc/how-to/images/setup-metabase/update_analytics_link.png rename to doc/how-to/metabase/images/setup-metabase/update_analytics_link.png diff --git a/doc/how-to/how-to-add-a-secret.md b/doc/how-to/secrets/how-to-add-a-secret.md similarity index 100% rename from doc/how-to/how-to-add-a-secret.md rename to doc/how-to/secrets/how-to-add-a-secret.md diff --git a/doc/how-to/how-to-add-a-team-secret.md b/doc/how-to/secrets/how-to-add-a-team-secret.md similarity index 100% rename from doc/how-to/how-to-add-a-team-secret.md rename to doc/how-to/secrets/how-to-add-a-team-secret.md diff --git a/doc/how-to/how-to-generate-a-secret.md b/doc/how-to/secrets/how-to-generate-a-secret.md similarity index 100% rename from doc/how-to/how-to-generate-a-secret.md rename to doc/how-to/secrets/how-to-generate-a-secret.md