From c3d9842a5f7365e7ac0e233b2527dc2eedddb8d5 Mon Sep 17 00:00:00 2001 From: Rory Doak Date: Mon, 4 Nov 2024 14:47:26 +0000 Subject: [PATCH 1/6] add ->> to application.type sql script --- hasura.planx.uk/metadata/tables.yaml | 176 ++++-------------- .../down.sql | 158 ++++++++++++++++ .../up.sql | 162 ++++++++++++++++ 3 files changed, 358 insertions(+), 138 deletions(-) create mode 100644 hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/down.sql create mode 100644 hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql diff --git a/hasura.planx.uk/metadata/tables.yaml b/hasura.planx.uk/metadata/tables.yaml index f1b5797f3f..ad8dad5728 100644 --- a/hasura.planx.uk/metadata/tables.yaml +++ b/hasura.planx.uk/metadata/tables.yaml @@ -89,9 +89,6 @@ status: _eq: online check: null -- table: - name: analytics_summary - schema: public - table: name: blpu_codes schema: public @@ -157,7 +154,7 @@ definition: enable_manual: false insert: - columns: "*" + columns: '*' retry_conf: interval_sec: 30 num_retries: 1 @@ -171,7 +168,7 @@ query_params: type: bops-submission template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/send-slack-notification" + url: '{{$base_url}}/webhooks/hasura/send-slack-notification' version: 2 - table: name: document_template @@ -225,7 +222,7 @@ definition: enable_manual: false insert: - columns: "*" + columns: '*' retry_conf: interval_sec: 30 num_retries: 1 @@ -239,7 +236,7 @@ query_params: type: email-submission template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/send-slack-notification" + url: '{{$base_url}}/webhooks/hasura/send-slack-notification' version: 2 - table: name: feedback @@ -483,9 +480,9 @@ forward_client_headers: false headers: - name: authorization - value: "{{HASURA_PLANX_API_KEY}}" + value: '{{HASURA_PLANX_API_KEY}}' timeout: 10 - url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" + url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' type: http - role: demoUser permission: @@ -529,9 +526,9 @@ forward_client_headers: false headers: - name: authorization - value: "{{HASURA_PLANX_API_KEY}}" + value: '{{HASURA_PLANX_API_KEY}}' timeout: 10 - url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" + url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' type: http - role: teamEditor permission: @@ -562,9 +559,9 @@ forward_client_headers: false headers: - name: authorization - value: "{{HASURA_PLANX_API_KEY}}" + value: '{{HASURA_PLANX_API_KEY}}' timeout: 10 - url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" + url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' type: http select_permissions: - role: api @@ -617,7 +614,7 @@ - 1 - 29 - 30 - comment: "For the demo user, we want to ensure they can only see their own flows, and flows from the Open Digital Planning [id = 30], Open Systems Lab [id = 1], and Templates [id = 29] team " + comment: 'For the demo user, we want to ensure they can only see their own flows, and flows from the Open Digital Planning [id = 30], Open Systems Lab [id = 1], and Templates [id = 29] team ' - role: platformAdmin permission: columns: @@ -698,9 +695,9 @@ forward_client_headers: false headers: - name: authorization - value: "{{HASURA_PLANX_API_KEY}}" + value: '{{HASURA_PLANX_API_KEY}}' timeout: 10 - url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" + url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' type: http - role: demoUser permission: @@ -740,9 +737,9 @@ forward_client_headers: false headers: - name: authorization - value: "{{HASURA_PLANX_API_KEY}}" + value: '{{HASURA_PLANX_API_KEY}}' timeout: 10 - url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" + url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' type: http - role: teamEditor permission: @@ -767,9 +764,9 @@ forward_client_headers: false headers: - name: authorization - value: "{{HASURA_PLANX_API_KEY}}" + value: '{{HASURA_PLANX_API_KEY}}' timeout: 10 - url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" + url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' type: http delete_permissions: - role: demoUser @@ -809,9 +806,9 @@ forward_client_headers: false headers: - name: authorization - value: "{{HASURA_PLANX_API_KEY}}" + value: '{{HASURA_PLANX_API_KEY}}' timeout: 10 - url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" + url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' type: http select_permissions: - role: demoUser @@ -849,9 +846,9 @@ forward_client_headers: false headers: - name: authorization - value: "{{HASURA_PLANX_API_KEY}}" + value: '{{HASURA_PLANX_API_KEY}}' timeout: 10 - url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" + url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' type: http - table: name: lowcal_sessions @@ -998,7 +995,7 @@ method: POST query_params: {} template_engine: Kriti - url: "{{$base_url}}/send-email/confirmation" + url: '{{$base_url}}/send-email/confirmation' version: 2 - name: setup_lowcal_expiry_events definition: @@ -1028,7 +1025,7 @@ method: POST query_params: {} template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/create-expiry-event" + url: '{{$base_url}}/webhooks/hasura/create-expiry-event' version: 2 - name: setup_lowcal_reminder_events definition: @@ -1058,7 +1055,7 @@ method: POST query_params: {} template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/create-reminder-event" + url: '{{$base_url}}/webhooks/hasura/create-reminder-event' version: 2 - table: name: operations @@ -1278,7 +1275,7 @@ definition: enable_manual: false insert: - columns: "*" + columns: '*' retry_conf: interval_sec: 10 num_retries: 3 @@ -1300,13 +1297,13 @@ method: POST query_params: {} template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/create-payment-expiry-events" + url: '{{$base_url}}/webhooks/hasura/create-payment-expiry-events' version: 2 - name: setup_payment_invitation_events definition: enable_manual: false insert: - columns: "*" + columns: '*' retry_conf: interval_sec: 10 num_retries: 3 @@ -1328,13 +1325,13 @@ method: POST query_params: {} template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/create-payment-invitation-events" + url: '{{$base_url}}/webhooks/hasura/create-payment-invitation-events' version: 2 - name: setup_payment_reminder_events definition: enable_manual: false insert: - columns: "*" + columns: '*' retry_conf: interval_sec: 10 num_retries: 3 @@ -1356,7 +1353,7 @@ method: POST query_params: {} template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/create-payment-reminder-events" + url: '{{$base_url}}/webhooks/hasura/create-payment-reminder-events' version: 2 - name: setup_payment_send_events definition: @@ -1385,7 +1382,7 @@ method: POST query_params: {} template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/create-payment-send-events" + url: '{{$base_url}}/webhooks/hasura/create-payment-send-events' version: 2 - table: name: payment_status @@ -1647,12 +1644,12 @@ definition: enable_manual: false insert: - columns: "*" + columns: '*' retry_conf: interval_sec: 30 num_retries: 1 timeout_sec: 60 - webhook: "{{HASURA_PLANX_API_URL}}" + webhook: '{{HASURA_PLANX_API_URL}}' headers: - name: authorization value_from_env: HASURA_PLANX_API_KEY @@ -1661,7 +1658,7 @@ query_params: type: s3-submission template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/send-slack-notification" + url: '{{$base_url}}/webhooks/hasura/send-slack-notification' version: 2 - table: name: sessions @@ -1770,105 +1767,8 @@ filter: {} comment: "" - table: - name: submission_services_summary + name: submission_services_summary_test schema: public - select_permissions: - - role: demoUser - permission: - columns: - - number_times_resumed - - sent_to_bops - - sent_to_email - - sent_to_s3_power_automate - - sent_to_uniform - - user_clicked_save - - user_invited_to_pay - - session_length_days - - bops_applications - - email_applications - - payment_requests - - payment_status - - s3_applications - - uniform_applications - - allow_list_answers - - application_declaration_connection - - draw_boundary_action - - find_property_action - - property_constraints_planning - - property_type - - proposal_project_type - - user_role - - service_slug - - session_id - - team_slug - - created_at - - submitted_at - filter: {} - comment: "For future, if this moves outside of the Flow to somewhere like Team, we should update 'demoUser' to only see submission data related to only their flows. " - - role: platformAdmin - permission: - columns: - - number_times_resumed - - sent_to_bops - - sent_to_email - - sent_to_s3_power_automate - - sent_to_uniform - - user_clicked_save - - user_invited_to_pay - - session_length_days - - bops_applications - - email_applications - - payment_requests - - payment_status - - s3_applications - - uniform_applications - - allow_list_answers - - application_declaration_connection - - draw_boundary_action - - find_property_action - - property_constraints_planning - - property_type - - proposal_project_type - - user_role - - service_slug - - session_id - - team_slug - - created_at - - submitted_at - filter: {} - comment: "" - - role: teamEditor - permission: - columns: - - number_times_resumed - - sent_to_bops - - sent_to_email - - sent_to_s3_power_automate - - sent_to_uniform - - user_clicked_save - - user_invited_to_pay - - session_length_days - - bops_applications - - email_applications - - payment_requests - - payment_status - - s3_applications - - uniform_applications - - allow_list_answers - - application_declaration_connection - - draw_boundary_action - - find_property_action - - property_constraints_planning - - property_type - - proposal_project_type - - user_role - - service_slug - - session_id - - team_slug - - created_at - - submitted_at - filter: {} - comment: "" - table: name: team_integrations schema: public @@ -2365,7 +2265,7 @@ - 29 - 30 - 32 - comment: "For the demo user, we want to ensure they can only see their own team [id = 32], and the teams: Open Digital Planning [id = 30], Open Systems Lab [id = 1], and Templates [id = 29] team " + comment: 'For the demo user, we want to ensure they can only see their own team [id = 32], and the teams: Open Digital Planning [id = 30], Open Systems Lab [id = 1], and Templates [id = 29] team ' - role: platformAdmin permission: columns: @@ -2482,7 +2382,7 @@ definition: enable_manual: false insert: - columns: "*" + columns: '*' retry_conf: interval_sec: 30 num_retries: 1 @@ -2496,7 +2396,7 @@ query_params: type: uniform-submission template_engine: Kriti - url: "{{$base_url}}/webhooks/hasura/send-slack-notification" + url: '{{$base_url}}/webhooks/hasura/send-slack-notification' version: 2 - table: name: user_roles diff --git a/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/down.sql b/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/down.sql new file mode 100644 index 0000000000..c11f361172 --- /dev/null +++ b/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/down.sql @@ -0,0 +1,158 @@ +CREATE OR REPLACE VIEW "public"."submission_services_summary" AS + WITH resumes_per_session AS ( + SELECT reconciliation_requests.session_id, + count(reconciliation_requests.id) AS number_times_resumed + FROM reconciliation_requests + GROUP BY reconciliation_requests.session_id + ), bops_agg AS ( + SELECT bops_applications.session_id, + json_agg(json_build_object('id', bops_applications.bops_id, 'submittedAt', bops_applications.created_at, 'destinationUrl', bops_applications.destination_url) ORDER BY bops_applications.created_at DESC) AS bops_applications + FROM bops_applications + GROUP BY bops_applications.session_id + ), email_agg AS ( + SELECT email_applications.session_id, + json_agg(json_build_object('id', email_applications.id, 'recipient', email_applications.recipient, 'submittedAt', email_applications.created_at) ORDER BY email_applications.created_at DESC) AS email_applications + FROM email_applications + GROUP BY email_applications.session_id + ), uniform_agg AS ( + SELECT uniform_applications.submission_reference, + json_agg(json_build_object('id', uniform_applications.idox_submission_id, 'submittedAt', uniform_applications.created_at) ORDER BY uniform_applications.created_at DESC) AS uniform_applications + FROM uniform_applications + GROUP BY uniform_applications.submission_reference + ), payment_requests_agg AS ( + SELECT payment_requests.session_id, + json_agg(json_build_object('id', payment_requests.id, 'createdAt', payment_requests.created_at, 'paidAt', payment_requests.paid_at, 'govpayPaymentId', payment_requests.govpay_payment_id) ORDER BY payment_requests.created_at DESC) AS payment_requests + FROM payment_requests + GROUP BY payment_requests.session_id + ), payment_status_agg AS ( + SELECT payment_status.session_id, + json_agg(json_build_object('govpayPaymentId', payment_status.payment_id, 'createdAt', payment_status.created_at, 'status', payment_status.status) ORDER BY payment_status.created_at DESC) AS payment_status + FROM payment_status + GROUP BY payment_status.session_id + ), s3_agg AS ( + SELECT s3_applications.session_id, + json_agg(json_build_object('id', s3_applications.id, 'submittedAt', s3_applications.created_at) ORDER BY s3_applications.created_at DESC) AS s3_applications + FROM s3_applications + GROUP BY s3_applications.session_id + ) + SELECT (ls.id)::text AS session_id, + t.slug AS team_slug, + f.slug AS service_slug, + ls.created_at, + ls.submitted_at, + ((ls.submitted_at)::date - (ls.created_at)::date) AS session_length_days, + ls.has_user_saved AS user_clicked_save, + rps.number_times_resumed, + ls.allow_list_answers, + (ls.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, + (ls.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, + (ls.allow_list_answers -> 'property.type'::text) AS property_type, + (ls.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, + (ls.allow_list_answers -> 'user.role'::text) AS user_role, + (ls.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, + CASE + WHEN (((pr.payment_requests)::jsonb IS NOT NULL) AND (jsonb_array_length((pr.payment_requests)::jsonb) > 0)) THEN true + ELSE false + END AS user_invited_to_pay, + pr.payment_requests, + ps.payment_status, + CASE + WHEN (((ba.bops_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((ba.bops_applications)::jsonb) > 0)) THEN true + ELSE false + END AS sent_to_bops, + ba.bops_applications, + CASE + WHEN (((ua.uniform_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((ua.uniform_applications)::jsonb) > 0)) THEN true + ELSE false + END AS sent_to_uniform, + ua.uniform_applications, + CASE + WHEN (((ea.email_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((ea.email_applications)::jsonb) > 0)) THEN true + ELSE false + END AS sent_to_email, + ea.email_applications, + (ls.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, + CASE + WHEN (((sa.s3_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((sa.s3_applications)::jsonb) > 0)) THEN true + ELSE false + END AS sent_to_s3_power_automate, + sa.s3_applications, + (ls.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, + (ls.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, + (ls.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, + (ls.allow_list_answers -> '_overrides'::text) AS overrides, + (ls.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, + (ls.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, + (ls.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, + (ls.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, + (ls.allow_list_answers -> 'application.type'::text) AS application_type + FROM (((((((((lowcal_sessions ls + LEFT JOIN flows f ON ((f.id = ls.flow_id))) + LEFT JOIN teams t ON ((t.id = f.team_id))) + LEFT JOIN resumes_per_session rps ON ((rps.session_id = (ls.id)::text))) + LEFT JOIN payment_requests_agg pr ON ((pr.session_id = ls.id))) + LEFT JOIN payment_status_agg ps ON ((ps.session_id = ls.id))) + LEFT JOIN bops_agg ba ON ((ba.session_id = (ls.id)::text))) + LEFT JOIN uniform_agg ua ON ((ua.submission_reference = (ls.id)::text))) + LEFT JOIN email_agg ea ON ((ea.session_id = ls.id))) + LEFT JOIN s3_agg sa ON ((sa.session_id = (ls.id)::text))) + WHERE ((f.slug IS NOT NULL) AND (t.slug IS NOT NULL)); + +CREATE OR REPLACE VIEW "public"."analytics_summary" AS + SELECT a.id AS analytics_id, + al.id AS analytics_log_id, + f.slug AS service_slug, + t.slug AS team_slug, + a.type AS analytics_type, + al.created_at AS analytics_log_created_at, + a.created_at AS analytics_created_at, + ((a.user_agent -> 'os'::text) ->> 'name'::text) AS operating_system, + ((a.user_agent -> 'browser'::text) ->> 'name'::text) AS browser, + ((a.user_agent -> 'platform'::text) ->> 'type'::text) AS platform, + a.referrer, + al.flow_direction, + (al.metadata ->> 'change'::text) AS change_metadata, + (al.metadata ->> 'back'::text) AS back_metadata, + (al.metadata ->> 'selectedUrls'::text) AS selected_urls, + (al.metadata ->> 'flag'::text) AS result_flag, + (al.metadata -> 'flagSet'::text) AS result_flagset, + ((al.metadata -> 'displayText'::text) ->> 'heading'::text) AS result_heading, + ((al.metadata -> 'displayText'::text) ->> 'description'::text) AS result_description, + (al.metadata -> 'helpTextUseful'::text) AS help_text_useful, + CASE + WHEN al.has_clicked_help THEN al.metadata + ELSE NULL::jsonb + END AS help_metadata, + al.user_exit AS is_user_exit, + al.node_type, + al.node_title, + al.has_clicked_help, + al.input_errors, + (date_part('epoch'::text, (al.next_log_created_at - al.created_at)))::numeric(10,1) AS time_spent_on_node_seconds, + a.ended_at AS analytics_ended_at, + ((date_part('epoch'::text, (a.ended_at - a.created_at)) / (60)::double precision))::numeric(10,1) AS time_spent_on_analytics_session_minutes, + al.node_id, + al.allow_list_answers, + (al.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, + (al.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, + (al.allow_list_answers -> 'property.type'::text) AS property_type, + (al.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, + (al.allow_list_answers -> 'user.role'::text) AS user_role, + (al.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, + (al.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, + (al.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, + (al.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, + (al.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, + (al.allow_list_answers -> '_overrides'::text) AS overrides, + (al.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, + (al.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, + (al.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, + (al.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, + (al.allow_list_answers -> 'application.type'::text) AS application_type + FROM (((analytics a + LEFT JOIN analytics_logs al ON ((a.id = al.analytics_id))) + LEFT JOIN flows f ON ((a.flow_id = f.id))) + LEFT JOIN teams t ON ((t.id = f.team_id))); + +GRANT SELECT ON "public"."analytics_summary" TO metabase_read_only; +GRANT SELECT ON "public"."submission_services_summary" TO metabase_read_only; \ No newline at end of file diff --git a/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql b/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql new file mode 100644 index 0000000000..040e6d924f --- /dev/null +++ b/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql @@ -0,0 +1,162 @@ +DROP VIEW IF EXISTS "public"."submission_services_summary"; + +CREATE VIEW "public"."submission_services_summary" AS + WITH resumes_per_session AS ( + SELECT reconciliation_requests.session_id, + count(reconciliation_requests.id) AS number_times_resumed + FROM reconciliation_requests + GROUP BY reconciliation_requests.session_id + ), bops_agg AS ( + SELECT bops_applications.session_id, + json_agg(json_build_object('id', bops_applications.bops_id, 'submittedAt', bops_applications.created_at, 'destinationUrl', bops_applications.destination_url) ORDER BY bops_applications.created_at DESC) AS bops_applications + FROM bops_applications + GROUP BY bops_applications.session_id + ), email_agg AS ( + SELECT email_applications.session_id, + json_agg(json_build_object('id', email_applications.id, 'recipient', email_applications.recipient, 'submittedAt', email_applications.created_at) ORDER BY email_applications.created_at DESC) AS email_applications + FROM email_applications + GROUP BY email_applications.session_id + ), uniform_agg AS ( + SELECT uniform_applications.submission_reference, + json_agg(json_build_object('id', uniform_applications.idox_submission_id, 'submittedAt', uniform_applications.created_at) ORDER BY uniform_applications.created_at DESC) AS uniform_applications + FROM uniform_applications + GROUP BY uniform_applications.submission_reference + ), payment_requests_agg AS ( + SELECT payment_requests.session_id, + json_agg(json_build_object('id', payment_requests.id, 'createdAt', payment_requests.created_at, 'paidAt', payment_requests.paid_at, 'govpayPaymentId', payment_requests.govpay_payment_id) ORDER BY payment_requests.created_at DESC) AS payment_requests + FROM payment_requests + GROUP BY payment_requests.session_id + ), payment_status_agg AS ( + SELECT payment_status.session_id, + json_agg(json_build_object('govpayPaymentId', payment_status.payment_id, 'createdAt', payment_status.created_at, 'status', payment_status.status) ORDER BY payment_status.created_at DESC) AS payment_status + FROM payment_status + GROUP BY payment_status.session_id + ), s3_agg AS ( + SELECT s3_applications.session_id, + json_agg(json_build_object('id', s3_applications.id, 'submittedAt', s3_applications.created_at) ORDER BY s3_applications.created_at DESC) AS s3_applications + FROM s3_applications + GROUP BY s3_applications.session_id + ) + SELECT (ls.id)::text AS session_id, + t.slug AS team_slug, + f.slug AS service_slug, + ls.created_at, + ls.submitted_at, + ((ls.submitted_at)::date - (ls.created_at)::date) AS session_length_days, + ls.has_user_saved AS user_clicked_save, + rps.number_times_resumed, + ls.allow_list_answers, + (ls.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, + (ls.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, + (ls.allow_list_answers -> 'property.type'::text) AS property_type, + (ls.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, + (ls.allow_list_answers -> 'user.role'::text) AS user_role, + (ls.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, + CASE + WHEN (((pr.payment_requests)::jsonb IS NOT NULL) AND (jsonb_array_length((pr.payment_requests)::jsonb) > 0)) THEN true + ELSE false + END AS user_invited_to_pay, + pr.payment_requests, + ps.payment_status, + CASE + WHEN (((ba.bops_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((ba.bops_applications)::jsonb) > 0)) THEN true + ELSE false + END AS sent_to_bops, + ba.bops_applications, + CASE + WHEN (((ua.uniform_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((ua.uniform_applications)::jsonb) > 0)) THEN true + ELSE false + END AS sent_to_uniform, + ua.uniform_applications, + CASE + WHEN (((ea.email_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((ea.email_applications)::jsonb) > 0)) THEN true + ELSE false + END AS sent_to_email, + ea.email_applications, + (ls.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, + CASE + WHEN (((sa.s3_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((sa.s3_applications)::jsonb) > 0)) THEN true + ELSE false + END AS sent_to_s3_power_automate, + sa.s3_applications, + (ls.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, + (ls.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, + (ls.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, + (ls.allow_list_answers -> '_overrides'::text) AS overrides, + (ls.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, + (ls.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, + (ls.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, + (ls.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, + (ls.allow_list_answers ->> 'application.type')::text AS application_type + FROM (((((((((lowcal_sessions ls + LEFT JOIN flows f ON ((f.id = ls.flow_id))) + LEFT JOIN teams t ON ((t.id = f.team_id))) + LEFT JOIN resumes_per_session rps ON ((rps.session_id = (ls.id)::text))) + LEFT JOIN payment_requests_agg pr ON ((pr.session_id = ls.id))) + LEFT JOIN payment_status_agg ps ON ((ps.session_id = ls.id))) + LEFT JOIN bops_agg ba ON ((ba.session_id = (ls.id)::text))) + LEFT JOIN uniform_agg ua ON ((ua.submission_reference = (ls.id)::text))) + LEFT JOIN email_agg ea ON ((ea.session_id = ls.id))) + LEFT JOIN s3_agg sa ON ((sa.session_id = (ls.id)::text))) + WHERE ((f.slug IS NOT NULL) AND (t.slug IS NOT NULL)); + +DROP VIEW IF EXISTS "public"."analytics_summary"; + +CREATE VIEW "public"."analytics_summary" AS + SELECT a.id AS analytics_id, + al.id AS analytics_log_id, + f.slug AS service_slug, + t.slug AS team_slug, + a.type AS analytics_type, + al.created_at AS analytics_log_created_at, + a.created_at AS analytics_created_at, + ((a.user_agent -> 'os'::text) ->> 'name'::text) AS operating_system, + ((a.user_agent -> 'browser'::text) ->> 'name'::text) AS browser, + ((a.user_agent -> 'platform'::text) ->> 'type'::text) AS platform, + a.referrer, + al.flow_direction, + (al.metadata ->> 'change'::text) AS change_metadata, + (al.metadata ->> 'back'::text) AS back_metadata, + (al.metadata ->> 'selectedUrls'::text) AS selected_urls, + (al.metadata ->> 'flag'::text) AS result_flag, + (al.metadata -> 'flagSet'::text) AS result_flagset, + ((al.metadata -> 'displayText'::text) ->> 'heading'::text) AS result_heading, + ((al.metadata -> 'displayText'::text) ->> 'description'::text) AS result_description, + (al.metadata -> 'helpTextUseful'::text) AS help_text_useful, + CASE + WHEN al.has_clicked_help THEN al.metadata + ELSE NULL::jsonb + END AS help_metadata, + al.user_exit AS is_user_exit, + al.node_type, + al.node_title, + al.has_clicked_help, + al.input_errors, + (date_part('epoch'::text, (al.next_log_created_at - al.created_at)))::numeric(10,1) AS time_spent_on_node_seconds, + a.ended_at AS analytics_ended_at, + ((date_part('epoch'::text, (a.ended_at - a.created_at)) / (60)::double precision))::numeric(10,1) AS time_spent_on_analytics_session_minutes, + al.node_id, + al.allow_list_answers, + (al.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, + (al.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, + (al.allow_list_answers -> 'property.type'::text) AS property_type, + (al.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, + (al.allow_list_answers -> 'user.role'::text) AS user_role, + (al.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, + (al.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, + (al.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, + (al.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, + (al.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, + (al.allow_list_answers -> '_overrides'::text) AS overrides, + (al.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, + (al.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, + (al.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, + (al.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, + (al.allow_list_answers ->> 'application.type')::text AS application_type + FROM (((analytics a + LEFT JOIN analytics_logs al ON ((a.id = al.analytics_id))) + LEFT JOIN flows f ON ((a.flow_id = f.id))) + LEFT JOIN teams t ON ((t.id = f.team_id))); + + GRANT SELECT ON "public"."analytics_summary" TO metabase_read_only; +GRANT SELECT ON "public"."submission_services_summary" TO metabase_read_only; From bcc4c60964983e84e05ede6cc9b09010754aded7 Mon Sep 17 00:00:00 2001 From: Rory Doak Date: Mon, 4 Nov 2024 15:01:49 +0000 Subject: [PATCH 2/6] ensure analytics_summary table is added to yaml --- hasura.planx.uk/metadata/tables.yaml | 5 +- .../1730732118559_run_sql_migration/down.sql | 59 +++++++++++++++++++ .../1730732118559_run_sql_migration/up.sql | 57 ++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 hasura.planx.uk/migrations/1730732118559_run_sql_migration/down.sql create mode 100644 hasura.planx.uk/migrations/1730732118559_run_sql_migration/up.sql diff --git a/hasura.planx.uk/metadata/tables.yaml b/hasura.planx.uk/metadata/tables.yaml index ad8dad5728..4a1ddc92e8 100644 --- a/hasura.planx.uk/metadata/tables.yaml +++ b/hasura.planx.uk/metadata/tables.yaml @@ -89,6 +89,9 @@ status: _eq: online check: null +- table: + name: analytics_summary + schema: public - table: name: blpu_codes schema: public @@ -1767,7 +1770,7 @@ filter: {} comment: "" - table: - name: submission_services_summary_test + name: submission_services_summary schema: public - table: name: team_integrations diff --git a/hasura.planx.uk/migrations/1730732118559_run_sql_migration/down.sql b/hasura.planx.uk/migrations/1730732118559_run_sql_migration/down.sql new file mode 100644 index 0000000000..212040559a --- /dev/null +++ b/hasura.planx.uk/migrations/1730732118559_run_sql_migration/down.sql @@ -0,0 +1,59 @@ +-- Could not auto-generate a down migration. +-- Please write an appropriate down migration for the SQL below: +-- DROP VIEW IF EXISTS "public"."analytics_summary"; +-- +-- CREATE VIEW "public"."analytics_summary" AS +-- SELECT a.id AS analytics_id, +-- al.id AS analytics_log_id, +-- f.slug AS service_slug, +-- t.slug AS team_slug, +-- a.type AS analytics_type, +-- al.created_at AS analytics_log_created_at, +-- a.created_at AS analytics_created_at, +-- ((a.user_agent -> 'os'::text) ->> 'name'::text) AS operating_system, +-- ((a.user_agent -> 'browser'::text) ->> 'name'::text) AS browser, +-- ((a.user_agent -> 'platform'::text) ->> 'type'::text) AS platform, +-- a.referrer, +-- al.flow_direction, +-- (al.metadata ->> 'change'::text) AS change_metadata, +-- (al.metadata ->> 'back'::text) AS back_metadata, +-- (al.metadata ->> 'selectedUrls'::text) AS selected_urls, +-- (al.metadata ->> 'flag'::text) AS result_flag, +-- (al.metadata -> 'flagSet'::text) AS result_flagset, +-- ((al.metadata -> 'displayText'::text) ->> 'heading'::text) AS result_heading, +-- ((al.metadata -> 'displayText'::text) ->> 'description'::text) AS result_description, +-- (al.metadata -> 'helpTextUseful'::text) AS help_text_useful, +-- CASE +-- WHEN al.has_clicked_help THEN al.metadata +-- ELSE NULL::jsonb +-- END AS help_metadata, +-- al.user_exit AS is_user_exit, +-- al.node_type, +-- al.node_title, +-- al.has_clicked_help, +-- al.input_errors, +-- (date_part('epoch'::text, (al.next_log_created_at - al.created_at)))::numeric(10,1) AS time_spent_on_node_seconds, +-- a.ended_at AS analytics_ended_at, +-- ((date_part('epoch'::text, (a.ended_at - a.created_at)) / (60)::double precision))::numeric(10,1) AS time_spent_on_analytics_session_minutes, +-- al.node_id, +-- al.allow_list_answers, +-- (al.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, +-- (al.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, +-- (al.allow_list_answers -> 'property.type'::text) AS property_type, +-- (al.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, +-- (al.allow_list_answers -> 'user.role'::text) AS user_role, +-- (al.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, +-- (al.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, +-- (al.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, +-- (al.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, +-- (al.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, +-- (al.allow_list_answers -> '_overrides'::text) AS overrides, +-- (al.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, +-- (al.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, +-- (al.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, +-- (al.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, +-- (al.allow_list_answers ->> 'application.type')::text AS application_type +-- FROM (((analytics a +-- LEFT JOIN analytics_logs al ON ((a.id = al.analytics_id))) +-- LEFT JOIN flows f ON ((a.flow_id = f.id))) +-- LEFT JOIN teams t ON ((t.id = f.team_id))); diff --git a/hasura.planx.uk/migrations/1730732118559_run_sql_migration/up.sql b/hasura.planx.uk/migrations/1730732118559_run_sql_migration/up.sql new file mode 100644 index 0000000000..053b8669b7 --- /dev/null +++ b/hasura.planx.uk/migrations/1730732118559_run_sql_migration/up.sql @@ -0,0 +1,57 @@ +DROP VIEW IF EXISTS "public"."analytics_summary"; + +CREATE VIEW "public"."analytics_summary" AS + SELECT a.id AS analytics_id, + al.id AS analytics_log_id, + f.slug AS service_slug, + t.slug AS team_slug, + a.type AS analytics_type, + al.created_at AS analytics_log_created_at, + a.created_at AS analytics_created_at, + ((a.user_agent -> 'os'::text) ->> 'name'::text) AS operating_system, + ((a.user_agent -> 'browser'::text) ->> 'name'::text) AS browser, + ((a.user_agent -> 'platform'::text) ->> 'type'::text) AS platform, + a.referrer, + al.flow_direction, + (al.metadata ->> 'change'::text) AS change_metadata, + (al.metadata ->> 'back'::text) AS back_metadata, + (al.metadata ->> 'selectedUrls'::text) AS selected_urls, + (al.metadata ->> 'flag'::text) AS result_flag, + (al.metadata -> 'flagSet'::text) AS result_flagset, + ((al.metadata -> 'displayText'::text) ->> 'heading'::text) AS result_heading, + ((al.metadata -> 'displayText'::text) ->> 'description'::text) AS result_description, + (al.metadata -> 'helpTextUseful'::text) AS help_text_useful, + CASE + WHEN al.has_clicked_help THEN al.metadata + ELSE NULL::jsonb + END AS help_metadata, + al.user_exit AS is_user_exit, + al.node_type, + al.node_title, + al.has_clicked_help, + al.input_errors, + (date_part('epoch'::text, (al.next_log_created_at - al.created_at)))::numeric(10,1) AS time_spent_on_node_seconds, + a.ended_at AS analytics_ended_at, + ((date_part('epoch'::text, (a.ended_at - a.created_at)) / (60)::double precision))::numeric(10,1) AS time_spent_on_analytics_session_minutes, + al.node_id, + al.allow_list_answers, + (al.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, + (al.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, + (al.allow_list_answers -> 'property.type'::text) AS property_type, + (al.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, + (al.allow_list_answers -> 'user.role'::text) AS user_role, + (al.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, + (al.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, + (al.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, + (al.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, + (al.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, + (al.allow_list_answers -> '_overrides'::text) AS overrides, + (al.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, + (al.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, + (al.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, + (al.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, + (al.allow_list_answers ->> 'application.type')::text AS application_type + FROM (((analytics a + LEFT JOIN analytics_logs al ON ((a.id = al.analytics_id))) + LEFT JOIN flows f ON ((a.flow_id = f.id))) + LEFT JOIN teams t ON ((t.id = f.team_id))); From ee55f8ed97f449a55fbdc3f6ca86535bc8a40df1 Mon Sep 17 00:00:00 2001 From: Rory Doak Date: Mon, 4 Nov 2024 15:02:35 +0000 Subject: [PATCH 3/6] delete extra scripts --- .../1730732118559_run_sql_migration/down.sql | 59 ------------------- .../1730732118559_run_sql_migration/up.sql | 57 ------------------ 2 files changed, 116 deletions(-) delete mode 100644 hasura.planx.uk/migrations/1730732118559_run_sql_migration/down.sql delete mode 100644 hasura.planx.uk/migrations/1730732118559_run_sql_migration/up.sql diff --git a/hasura.planx.uk/migrations/1730732118559_run_sql_migration/down.sql b/hasura.planx.uk/migrations/1730732118559_run_sql_migration/down.sql deleted file mode 100644 index 212040559a..0000000000 --- a/hasura.planx.uk/migrations/1730732118559_run_sql_migration/down.sql +++ /dev/null @@ -1,59 +0,0 @@ --- Could not auto-generate a down migration. --- Please write an appropriate down migration for the SQL below: --- DROP VIEW IF EXISTS "public"."analytics_summary"; --- --- CREATE VIEW "public"."analytics_summary" AS --- SELECT a.id AS analytics_id, --- al.id AS analytics_log_id, --- f.slug AS service_slug, --- t.slug AS team_slug, --- a.type AS analytics_type, --- al.created_at AS analytics_log_created_at, --- a.created_at AS analytics_created_at, --- ((a.user_agent -> 'os'::text) ->> 'name'::text) AS operating_system, --- ((a.user_agent -> 'browser'::text) ->> 'name'::text) AS browser, --- ((a.user_agent -> 'platform'::text) ->> 'type'::text) AS platform, --- a.referrer, --- al.flow_direction, --- (al.metadata ->> 'change'::text) AS change_metadata, --- (al.metadata ->> 'back'::text) AS back_metadata, --- (al.metadata ->> 'selectedUrls'::text) AS selected_urls, --- (al.metadata ->> 'flag'::text) AS result_flag, --- (al.metadata -> 'flagSet'::text) AS result_flagset, --- ((al.metadata -> 'displayText'::text) ->> 'heading'::text) AS result_heading, --- ((al.metadata -> 'displayText'::text) ->> 'description'::text) AS result_description, --- (al.metadata -> 'helpTextUseful'::text) AS help_text_useful, --- CASE --- WHEN al.has_clicked_help THEN al.metadata --- ELSE NULL::jsonb --- END AS help_metadata, --- al.user_exit AS is_user_exit, --- al.node_type, --- al.node_title, --- al.has_clicked_help, --- al.input_errors, --- (date_part('epoch'::text, (al.next_log_created_at - al.created_at)))::numeric(10,1) AS time_spent_on_node_seconds, --- a.ended_at AS analytics_ended_at, --- ((date_part('epoch'::text, (a.ended_at - a.created_at)) / (60)::double precision))::numeric(10,1) AS time_spent_on_analytics_session_minutes, --- al.node_id, --- al.allow_list_answers, --- (al.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, --- (al.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, --- (al.allow_list_answers -> 'property.type'::text) AS property_type, --- (al.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, --- (al.allow_list_answers -> 'user.role'::text) AS user_role, --- (al.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, --- (al.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, --- (al.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, --- (al.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, --- (al.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, --- (al.allow_list_answers -> '_overrides'::text) AS overrides, --- (al.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, --- (al.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, --- (al.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, --- (al.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, --- (al.allow_list_answers ->> 'application.type')::text AS application_type --- FROM (((analytics a --- LEFT JOIN analytics_logs al ON ((a.id = al.analytics_id))) --- LEFT JOIN flows f ON ((a.flow_id = f.id))) --- LEFT JOIN teams t ON ((t.id = f.team_id))); diff --git a/hasura.planx.uk/migrations/1730732118559_run_sql_migration/up.sql b/hasura.planx.uk/migrations/1730732118559_run_sql_migration/up.sql deleted file mode 100644 index 053b8669b7..0000000000 --- a/hasura.planx.uk/migrations/1730732118559_run_sql_migration/up.sql +++ /dev/null @@ -1,57 +0,0 @@ -DROP VIEW IF EXISTS "public"."analytics_summary"; - -CREATE VIEW "public"."analytics_summary" AS - SELECT a.id AS analytics_id, - al.id AS analytics_log_id, - f.slug AS service_slug, - t.slug AS team_slug, - a.type AS analytics_type, - al.created_at AS analytics_log_created_at, - a.created_at AS analytics_created_at, - ((a.user_agent -> 'os'::text) ->> 'name'::text) AS operating_system, - ((a.user_agent -> 'browser'::text) ->> 'name'::text) AS browser, - ((a.user_agent -> 'platform'::text) ->> 'type'::text) AS platform, - a.referrer, - al.flow_direction, - (al.metadata ->> 'change'::text) AS change_metadata, - (al.metadata ->> 'back'::text) AS back_metadata, - (al.metadata ->> 'selectedUrls'::text) AS selected_urls, - (al.metadata ->> 'flag'::text) AS result_flag, - (al.metadata -> 'flagSet'::text) AS result_flagset, - ((al.metadata -> 'displayText'::text) ->> 'heading'::text) AS result_heading, - ((al.metadata -> 'displayText'::text) ->> 'description'::text) AS result_description, - (al.metadata -> 'helpTextUseful'::text) AS help_text_useful, - CASE - WHEN al.has_clicked_help THEN al.metadata - ELSE NULL::jsonb - END AS help_metadata, - al.user_exit AS is_user_exit, - al.node_type, - al.node_title, - al.has_clicked_help, - al.input_errors, - (date_part('epoch'::text, (al.next_log_created_at - al.created_at)))::numeric(10,1) AS time_spent_on_node_seconds, - a.ended_at AS analytics_ended_at, - ((date_part('epoch'::text, (a.ended_at - a.created_at)) / (60)::double precision))::numeric(10,1) AS time_spent_on_analytics_session_minutes, - al.node_id, - al.allow_list_answers, - (al.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, - (al.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, - (al.allow_list_answers -> 'property.type'::text) AS property_type, - (al.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, - (al.allow_list_answers -> 'user.role'::text) AS user_role, - (al.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, - (al.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, - (al.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, - (al.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, - (al.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, - (al.allow_list_answers -> '_overrides'::text) AS overrides, - (al.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, - (al.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, - (al.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, - (al.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, - (al.allow_list_answers ->> 'application.type')::text AS application_type - FROM (((analytics a - LEFT JOIN analytics_logs al ON ((a.id = al.analytics_id))) - LEFT JOIN flows f ON ((a.flow_id = f.id))) - LEFT JOIN teams t ON ((t.id = f.team_id))); From b64439b1f5704b06a88e3931bafba74bf1c09b34 Mon Sep 17 00:00:00 2001 From: Rory Doak Date: Mon, 4 Nov 2024 15:09:00 +0000 Subject: [PATCH 4/6] rever tables.yaml changes --- hasura.planx.uk/metadata/tables.yaml | 171 +++++++++++++++++++++------ 1 file changed, 134 insertions(+), 37 deletions(-) diff --git a/hasura.planx.uk/metadata/tables.yaml b/hasura.planx.uk/metadata/tables.yaml index 4a1ddc92e8..f1b5797f3f 100644 --- a/hasura.planx.uk/metadata/tables.yaml +++ b/hasura.planx.uk/metadata/tables.yaml @@ -157,7 +157,7 @@ definition: enable_manual: false insert: - columns: '*' + columns: "*" retry_conf: interval_sec: 30 num_retries: 1 @@ -171,7 +171,7 @@ query_params: type: bops-submission template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/send-slack-notification' + url: "{{$base_url}}/webhooks/hasura/send-slack-notification" version: 2 - table: name: document_template @@ -225,7 +225,7 @@ definition: enable_manual: false insert: - columns: '*' + columns: "*" retry_conf: interval_sec: 30 num_retries: 1 @@ -239,7 +239,7 @@ query_params: type: email-submission template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/send-slack-notification' + url: "{{$base_url}}/webhooks/hasura/send-slack-notification" version: 2 - table: name: feedback @@ -483,9 +483,9 @@ forward_client_headers: false headers: - name: authorization - value: '{{HASURA_PLANX_API_KEY}}' + value: "{{HASURA_PLANX_API_KEY}}" timeout: 10 - url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' + url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" type: http - role: demoUser permission: @@ -529,9 +529,9 @@ forward_client_headers: false headers: - name: authorization - value: '{{HASURA_PLANX_API_KEY}}' + value: "{{HASURA_PLANX_API_KEY}}" timeout: 10 - url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' + url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" type: http - role: teamEditor permission: @@ -562,9 +562,9 @@ forward_client_headers: false headers: - name: authorization - value: '{{HASURA_PLANX_API_KEY}}' + value: "{{HASURA_PLANX_API_KEY}}" timeout: 10 - url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' + url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" type: http select_permissions: - role: api @@ -617,7 +617,7 @@ - 1 - 29 - 30 - comment: 'For the demo user, we want to ensure they can only see their own flows, and flows from the Open Digital Planning [id = 30], Open Systems Lab [id = 1], and Templates [id = 29] team ' + comment: "For the demo user, we want to ensure they can only see their own flows, and flows from the Open Digital Planning [id = 30], Open Systems Lab [id = 1], and Templates [id = 29] team " - role: platformAdmin permission: columns: @@ -698,9 +698,9 @@ forward_client_headers: false headers: - name: authorization - value: '{{HASURA_PLANX_API_KEY}}' + value: "{{HASURA_PLANX_API_KEY}}" timeout: 10 - url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' + url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" type: http - role: demoUser permission: @@ -740,9 +740,9 @@ forward_client_headers: false headers: - name: authorization - value: '{{HASURA_PLANX_API_KEY}}' + value: "{{HASURA_PLANX_API_KEY}}" timeout: 10 - url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' + url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" type: http - role: teamEditor permission: @@ -767,9 +767,9 @@ forward_client_headers: false headers: - name: authorization - value: '{{HASURA_PLANX_API_KEY}}' + value: "{{HASURA_PLANX_API_KEY}}" timeout: 10 - url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' + url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" type: http delete_permissions: - role: demoUser @@ -809,9 +809,9 @@ forward_client_headers: false headers: - name: authorization - value: '{{HASURA_PLANX_API_KEY}}' + value: "{{HASURA_PLANX_API_KEY}}" timeout: 10 - url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' + url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" type: http select_permissions: - role: demoUser @@ -849,9 +849,9 @@ forward_client_headers: false headers: - name: authorization - value: '{{HASURA_PLANX_API_KEY}}' + value: "{{HASURA_PLANX_API_KEY}}" timeout: 10 - url: '{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html' + url: "{{HASURA_PLANX_API_URL}}/webhooks/hasura/validate-input/jsonb/clean-html" type: http - table: name: lowcal_sessions @@ -998,7 +998,7 @@ method: POST query_params: {} template_engine: Kriti - url: '{{$base_url}}/send-email/confirmation' + url: "{{$base_url}}/send-email/confirmation" version: 2 - name: setup_lowcal_expiry_events definition: @@ -1028,7 +1028,7 @@ method: POST query_params: {} template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/create-expiry-event' + url: "{{$base_url}}/webhooks/hasura/create-expiry-event" version: 2 - name: setup_lowcal_reminder_events definition: @@ -1058,7 +1058,7 @@ method: POST query_params: {} template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/create-reminder-event' + url: "{{$base_url}}/webhooks/hasura/create-reminder-event" version: 2 - table: name: operations @@ -1278,7 +1278,7 @@ definition: enable_manual: false insert: - columns: '*' + columns: "*" retry_conf: interval_sec: 10 num_retries: 3 @@ -1300,13 +1300,13 @@ method: POST query_params: {} template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/create-payment-expiry-events' + url: "{{$base_url}}/webhooks/hasura/create-payment-expiry-events" version: 2 - name: setup_payment_invitation_events definition: enable_manual: false insert: - columns: '*' + columns: "*" retry_conf: interval_sec: 10 num_retries: 3 @@ -1328,13 +1328,13 @@ method: POST query_params: {} template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/create-payment-invitation-events' + url: "{{$base_url}}/webhooks/hasura/create-payment-invitation-events" version: 2 - name: setup_payment_reminder_events definition: enable_manual: false insert: - columns: '*' + columns: "*" retry_conf: interval_sec: 10 num_retries: 3 @@ -1356,7 +1356,7 @@ method: POST query_params: {} template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/create-payment-reminder-events' + url: "{{$base_url}}/webhooks/hasura/create-payment-reminder-events" version: 2 - name: setup_payment_send_events definition: @@ -1385,7 +1385,7 @@ method: POST query_params: {} template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/create-payment-send-events' + url: "{{$base_url}}/webhooks/hasura/create-payment-send-events" version: 2 - table: name: payment_status @@ -1647,12 +1647,12 @@ definition: enable_manual: false insert: - columns: '*' + columns: "*" retry_conf: interval_sec: 30 num_retries: 1 timeout_sec: 60 - webhook: '{{HASURA_PLANX_API_URL}}' + webhook: "{{HASURA_PLANX_API_URL}}" headers: - name: authorization value_from_env: HASURA_PLANX_API_KEY @@ -1661,7 +1661,7 @@ query_params: type: s3-submission template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/send-slack-notification' + url: "{{$base_url}}/webhooks/hasura/send-slack-notification" version: 2 - table: name: sessions @@ -1772,6 +1772,103 @@ - table: name: submission_services_summary schema: public + select_permissions: + - role: demoUser + permission: + columns: + - number_times_resumed + - sent_to_bops + - sent_to_email + - sent_to_s3_power_automate + - sent_to_uniform + - user_clicked_save + - user_invited_to_pay + - session_length_days + - bops_applications + - email_applications + - payment_requests + - payment_status + - s3_applications + - uniform_applications + - allow_list_answers + - application_declaration_connection + - draw_boundary_action + - find_property_action + - property_constraints_planning + - property_type + - proposal_project_type + - user_role + - service_slug + - session_id + - team_slug + - created_at + - submitted_at + filter: {} + comment: "For future, if this moves outside of the Flow to somewhere like Team, we should update 'demoUser' to only see submission data related to only their flows. " + - role: platformAdmin + permission: + columns: + - number_times_resumed + - sent_to_bops + - sent_to_email + - sent_to_s3_power_automate + - sent_to_uniform + - user_clicked_save + - user_invited_to_pay + - session_length_days + - bops_applications + - email_applications + - payment_requests + - payment_status + - s3_applications + - uniform_applications + - allow_list_answers + - application_declaration_connection + - draw_boundary_action + - find_property_action + - property_constraints_planning + - property_type + - proposal_project_type + - user_role + - service_slug + - session_id + - team_slug + - created_at + - submitted_at + filter: {} + comment: "" + - role: teamEditor + permission: + columns: + - number_times_resumed + - sent_to_bops + - sent_to_email + - sent_to_s3_power_automate + - sent_to_uniform + - user_clicked_save + - user_invited_to_pay + - session_length_days + - bops_applications + - email_applications + - payment_requests + - payment_status + - s3_applications + - uniform_applications + - allow_list_answers + - application_declaration_connection + - draw_boundary_action + - find_property_action + - property_constraints_planning + - property_type + - proposal_project_type + - user_role + - service_slug + - session_id + - team_slug + - created_at + - submitted_at + filter: {} + comment: "" - table: name: team_integrations schema: public @@ -2268,7 +2365,7 @@ - 29 - 30 - 32 - comment: 'For the demo user, we want to ensure they can only see their own team [id = 32], and the teams: Open Digital Planning [id = 30], Open Systems Lab [id = 1], and Templates [id = 29] team ' + comment: "For the demo user, we want to ensure they can only see their own team [id = 32], and the teams: Open Digital Planning [id = 30], Open Systems Lab [id = 1], and Templates [id = 29] team " - role: platformAdmin permission: columns: @@ -2385,7 +2482,7 @@ definition: enable_manual: false insert: - columns: '*' + columns: "*" retry_conf: interval_sec: 30 num_retries: 1 @@ -2399,7 +2496,7 @@ query_params: type: uniform-submission template_engine: Kriti - url: '{{$base_url}}/webhooks/hasura/send-slack-notification' + url: "{{$base_url}}/webhooks/hasura/send-slack-notification" version: 2 - table: name: user_roles From d7c79b0551bb555166071f7d75aba976a6da79eb Mon Sep 17 00:00:00 2001 From: Rory Doak Date: Mon, 4 Nov 2024 15:46:03 +0000 Subject: [PATCH 5/6] alter sql to pull value out of array --- .../1730730955039_application_type_as_text_in_view/up.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql b/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql index 040e6d924f..37c0dcec53 100644 --- a/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql +++ b/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql @@ -87,7 +87,7 @@ CREATE VIEW "public"."submission_services_summary" AS (ls.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, (ls.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, (ls.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, - (ls.allow_list_answers ->> 'application.type')::text AS application_type + (ls.allow_list_answers -> 'application.type' -> 0)::text AS application_type FROM (((((((((lowcal_sessions ls LEFT JOIN flows f ON ((f.id = ls.flow_id))) LEFT JOIN teams t ON ((t.id = f.team_id))) @@ -152,7 +152,7 @@ CREATE VIEW "public"."analytics_summary" AS (al.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, (al.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, (al.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, - (al.allow_list_answers ->> 'application.type')::text AS application_type + (al.allow_list_answers -> 'application.type' -> 0)::text AS application_type FROM (((analytics a LEFT JOIN analytics_logs al ON ((a.id = al.analytics_id))) LEFT JOIN flows f ON ((a.flow_id = f.id))) From 51703ad0bed22798549ce8f83069da72cc515ba3 Mon Sep 17 00:00:00 2001 From: Rory Doak Date: Tue, 5 Nov 2024 09:34:38 +0000 Subject: [PATCH 6/6] expand fix to other columns --- .../up.sql | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql b/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql index 37c0dcec53..99ab6b85ff 100644 --- a/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql +++ b/hasura.planx.uk/migrations/1730730955039_application_type_as_text_in_view/up.sql @@ -46,12 +46,12 @@ CREATE VIEW "public"."submission_services_summary" AS ls.has_user_saved AS user_clicked_save, rps.number_times_resumed, ls.allow_list_answers, - (ls.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, - (ls.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, - (ls.allow_list_answers -> 'property.type'::text) AS property_type, - (ls.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, - (ls.allow_list_answers -> 'user.role'::text) AS user_role, - (ls.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, + (ls.allow_list_answers -> 'proposal.projectType')::text AS proposal_project_type, + (ls.allow_list_answers -> 'application.declaration.connection')::text AS application_declaration_connection, + (ls.allow_list_answers -> 'property.type')::text AS property_type, + (ls.allow_list_answers -> 'drawBoundary.action')::text AS draw_boundary_action, + (ls.allow_list_answers -> 'user.role')::text AS user_role, + (ls.allow_list_answers -> 'property.constraints.planning')::text AS property_constraints_planning, CASE WHEN (((pr.payment_requests)::jsonb IS NOT NULL) AND (jsonb_array_length((pr.payment_requests)::jsonb) > 0)) THEN true ELSE false @@ -73,20 +73,20 @@ CREATE VIEW "public"."submission_services_summary" AS ELSE false END AS sent_to_email, ea.email_applications, - (ls.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, + (ls.allow_list_answers -> 'findProperty.action')::text AS find_property_action, CASE WHEN (((sa.s3_applications)::jsonb IS NOT NULL) AND (jsonb_array_length((sa.s3_applications)::jsonb) > 0)) THEN true ELSE false END AS sent_to_s3_power_automate, sa.s3_applications, - (ls.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, - (ls.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, - (ls.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, - (ls.allow_list_answers -> '_overrides'::text) AS overrides, - (ls.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, - (ls.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, - (ls.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, - (ls.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, + (ls.allow_list_answers -> 'usedFOIYNPP')::text AS used_foiynpp, + (ls.allow_list_answers -> 'propertyInformation.action')::text AS property_information_action, + (ls.allow_list_answers -> 'planningConstraints.action')::text AS planning_constraints_action, + (ls.allow_list_answers -> '_overrides')::text AS overrides, + (ls.allow_list_answers -> 'rab.exitReason')::text AS rab_exit_reason, + (ls.allow_list_answers -> 'service.type')::text AS pre_app_service_type, + (ls.allow_list_answers -> 'application.information.harmful')::text AS pre_app_harmful_info, + (ls.allow_list_answers -> 'application.information.sensitive')::text AS pre_app_sensitive_info, (ls.allow_list_answers -> 'application.type' -> 0)::text AS application_type FROM (((((((((lowcal_sessions ls LEFT JOIN flows f ON ((f.id = ls.flow_id))) @@ -115,14 +115,14 @@ CREATE VIEW "public"."analytics_summary" AS ((a.user_agent -> 'platform'::text) ->> 'type'::text) AS platform, a.referrer, al.flow_direction, - (al.metadata ->> 'change'::text) AS change_metadata, - (al.metadata ->> 'back'::text) AS back_metadata, - (al.metadata ->> 'selectedUrls'::text) AS selected_urls, - (al.metadata ->> 'flag'::text) AS result_flag, - (al.metadata -> 'flagSet'::text) AS result_flagset, - ((al.metadata -> 'displayText'::text) ->> 'heading'::text) AS result_heading, - ((al.metadata -> 'displayText'::text) ->> 'description'::text) AS result_description, - (al.metadata -> 'helpTextUseful'::text) AS help_text_useful, + (al.metadata ->> 'change')::text AS change_metadata, + (al.metadata ->> 'back')::text AS back_metadata, + (al.metadata ->> 'selectedUrls')::text AS selected_urls, + (al.metadata ->> 'flag')::text AS result_flag, + (al.metadata -> 'flagSet')::text AS result_flagset, + ((al.metadata -> 'displayText'::text) ->> 'heading')::text AS result_heading, + ((al.metadata -> 'displayText'::text) ->> 'description')::text AS result_description, + (al.metadata -> 'helpTextUseful')::text AS help_text_useful, CASE WHEN al.has_clicked_help THEN al.metadata ELSE NULL::jsonb @@ -137,21 +137,21 @@ CREATE VIEW "public"."analytics_summary" AS ((date_part('epoch'::text, (a.ended_at - a.created_at)) / (60)::double precision))::numeric(10,1) AS time_spent_on_analytics_session_minutes, al.node_id, al.allow_list_answers, - (al.allow_list_answers -> 'proposal.projectType'::text) AS proposal_project_type, - (al.allow_list_answers -> 'application.declaration.connection'::text) AS application_declaration_connection, - (al.allow_list_answers -> 'property.type'::text) AS property_type, - (al.allow_list_answers -> 'drawBoundary.action'::text) AS draw_boundary_action, - (al.allow_list_answers -> 'user.role'::text) AS user_role, - (al.allow_list_answers -> 'property.constraints.planning'::text) AS property_constraints_planning, - (al.allow_list_answers -> 'findProperty.action'::text) AS find_property_action, - (al.allow_list_answers -> 'usedFOIYNPP'::text) AS used_foiynpp, - (al.allow_list_answers -> 'propertyInformation.action'::text) AS property_information_action, - (al.allow_list_answers -> 'planningConstraints.action'::text) AS planning_constraints_action, - (al.allow_list_answers -> '_overrides'::text) AS overrides, - (al.allow_list_answers -> 'rab.exitReason'::text) AS rab_exit_reason, - (al.allow_list_answers -> 'service.type'::text) AS pre_app_service_type, - (al.allow_list_answers -> 'application.information.harmful'::text) AS pre_app_harmful_info, - (al.allow_list_answers -> 'application.information.sensitive'::text) AS pre_app_sensitive_info, + (al.allow_list_answers -> 'proposal.projectType')::text AS proposal_project_type, + (al.allow_list_answers -> 'application.declaration.connection')::text AS application_declaration_connection, + (al.allow_list_answers -> 'property.type')::text AS property_type, + (al.allow_list_answers -> 'drawBoundary.action')::text AS draw_boundary_action, + (al.allow_list_answers -> 'user.role')::text AS user_role, + (al.allow_list_answers -> 'property.constraints.planning')::text AS property_constraints_planning, + (al.allow_list_answers -> 'findProperty.action')::text AS find_property_action, + (al.allow_list_answers -> 'usedFOIYNPP')::text AS used_foiynpp, + (al.allow_list_answers -> 'propertyInformation.action')::text AS property_information_action, + (al.allow_list_answers -> 'planningConstraints.action')::text AS planning_constraints_action, + (al.allow_list_answers -> '_overrides')::text AS overrides, + (al.allow_list_answers -> 'rab.exitReason')::text AS rab_exit_reason, + (al.allow_list_answers -> 'service.type')::text AS pre_app_service_type, + (al.allow_list_answers -> 'application.information.harmful')::text AS pre_app_harmful_info, + (al.allow_list_answers -> 'application.information.sensitive')::text AS pre_app_sensitive_info, (al.allow_list_answers -> 'application.type' -> 0)::text AS application_type FROM (((analytics a LEFT JOIN analytics_logs al ON ((a.id = al.analytics_id)))