refactor: store analytics_logs allow_list_answers from arrays of objects to object

editor.planx.uk/src/pages/FlowEditor/lib/analyticsProvider.tsx

@@ -31,7 +31,9 @@ const ALLOW_LIST = [
   "drawBoundary.action",
   "user.role",
   "property.constraints.planning",
-];
+] as const;
+
+type AllowListKey = (typeof ALLOW_LIST)[number];
 
 export type HelpClickMetadata = Record<string, string>;
 type SelectedUrlsMetadata = Record<"selectedUrls", string[]>;
@@ -206,7 +208,6 @@ export const AnalyticsProvider: React.FC<{ children: React.ReactNode }> = ({
     const metadata: NodeMetadata = getNodeMetadata(nodeToTrack, nodeId);
     const nodeType = nodeToTrack?.type ? TYPES[nodeToTrack.type] : null;
     const nodeTitle = extractNodeTitle(nodeToTrack);
-    const nodeFn = nodeToTrack?.data?.fn || nodeToTrack?.data?.val;
 
     const result = await insertNewAnalyticsLog(
       logDirection,
@@ -215,7 +216,6 @@ export const AnalyticsProvider: React.FC<{ children: React.ReactNode }> = ({
       nodeType,
       nodeTitle,
       nodeId,
-      nodeFn,
     );
 
     const { id, created_at: newLogCreatedAt } =
@@ -248,7 +248,6 @@ export const AnalyticsProvider: React.FC<{ children: React.ReactNode }> = ({
     nodeType: string | null,
     nodeTitle: string,
     nodeId: string | null,
-    nodeFn: string | null,
   ) {
     const result = await publicClient.mutate({
       mutation: gql`
@@ -259,7 +258,6 @@ export const AnalyticsProvider: React.FC<{ children: React.ReactNode }> = ({
           $node_type: String
           $node_title: String
           $node_id: String
-          $node_fn: String
         ) {
           insert_analytics_logs_one(
             object: {
@@ -270,7 +268,6 @@ export const AnalyticsProvider: React.FC<{ children: React.ReactNode }> = ({
               node_type: $node_type
               node_title: $node_title
               node_id: $node_id
-              node_fn: $node_fn
             }
           ) {
             id
@@ -285,7 +282,6 @@ export const AnalyticsProvider: React.FC<{ children: React.ReactNode }> = ({
         node_type: nodeType,
         node_title: nodeTitle,
         node_id: nodeId,
-        node_fn: nodeFn,
       },
     });
     return result;
@@ -497,32 +493,46 @@ export const AnalyticsProvider: React.FC<{ children: React.ReactNode }> = ({
   function getAllowListAnswers(
     nodeId: string,
     breadcrumb: Store.userData,
-  ): Record<string, unknown>[] | undefined {
+  ): Partial<Record<AllowListKey, any>> | undefined {
     const answers = getAnswers(nodeId);
     const data = getData(breadcrumb);
 
-    const allowListAnswers = [...answers, ...data];
-    if (!allowListAnswers.length) return;
+    if (!answers && !data) return;
+    const allowListAnswers = Object.assign({}, answers, data);
 
     return allowListAnswers;
   }
 
+  /**
+   * Check whether the key is in the ALLOW_LIST and ensure it's of the correct
+   * type to avoid repeated casting.
+   */
+  function isAllowListKey(key: any): key is AllowListKey {
+    return (ALLOW_LIST as readonly string[]).includes(key);
+  }
+
   /**
    * Extract allowlist answers from user answers
-   * e.g. from Checklist or Question components
+   * e.g., from Checklist or Question components
    */
-  function getAnswers(nodeId: string) {
+  function getAnswers(
+    nodeId: string,
+  ): Partial<Record<AllowListKey, any>> | undefined {
     const { data } = flow[nodeId];
-    const nodeFn: string = data?.fn || data?.val;
-    if (!nodeFn || !ALLOW_LIST.includes(nodeFn)) return [];
+    const nodeFn: string | undefined = data?.fn || data?.val;
+
+    if (!nodeFn || !isAllowListKey(nodeFn)) return;
 
     const answerIds = breadcrumbs[nodeId]?.answers;
-    if (!answerIds) return [];
+    if (!answerIds) return;
 
     const answerValues = answerIds.map((answerId) => flow[answerId]?.data?.val);
+    const filteredAnswerValues = answerValues.filter(Boolean);
+    if (!filteredAnswerValues.length) return;
 
-    // Match data structure of `allow_list_answers` column
-    const answers = [{ [nodeFn]: answerValues }];
+    const answers: Partial<Record<AllowListKey, string[]>> = {
+      [nodeFn]: filteredAnswerValues,
+    };
 
     return answers;
   }
@@ -531,14 +541,19 @@ export const AnalyticsProvider: React.FC<{ children: React.ReactNode }> = ({
    * Extract allowlist answers from breadcrumb data
    * e.g. data set automatically by components such as DrawBoundary
    */
-  function getData(breadcrumb: Store.userData) {
+  function getData(
+    breadcrumb: Store.userData,
+  ): Partial<Record<AllowListKey, any>> | undefined {
     const dataSetByNode = breadcrumb.data;
-    if (!dataSetByNode) return [];
+    if (!dataSetByNode) return;
 
-    const answerValues = Object.entries(dataSetByNode)
-      .filter(([key, value]) => ALLOW_LIST.includes(key) && Boolean(value))
+    const filteredEntries = Object.entries(dataSetByNode)
+      .filter(([key, value]) => isAllowListKey(key) && Boolean(value))
       .map(([key, value]) => ({ [key]: value }));
 
+    if (!filteredEntries.length) return;
+    const answerValues = Object.assign({}, ...filteredEntries);
+
     return answerValues;
   }
 

hasura.planx.uk/metadata/tables.yaml

@@ -40,7 +40,6 @@
           - id
           - input_errors
           - metadata
-          - node_fn
           - node_id
           - node_title
           - node_type

hasura.planx.uk/migrations/1711643574155_update_analytics_logs_allow_list_answers_array_to_object/down.sql

@@ -0,0 +1,72 @@
+-- Update the data structure of allow_list_answers to revert it to an array of objects
+
+UPDATE public.analytics_logs
+SET allow_list_answers = jsonb_build_array(allow_list_answers)
+WHERE jsonb_typeof(allow_list_answers) != 'array';
+
+-- Previous instance of view from hasura.planx.uk/migrations/1711457331702_alter_view_public_analytics_summary_split_allow_list_answers_into_columns/up.sql
+DROP VIEW public.analytics_summary;
+
+CREATE
+OR REPLACE VIEW public.analytics_summary AS
+select
+	a.id as analytics_id,
+	al.id as analytics_log_id,
+	f.slug as service_slug,
+	t.slug as team_slug,
+	a.type as analytics_type,
+	al.created_at as analytics_log_created_at,
+	a.created_at as analytics_created_at,
+	(user_agent -> 'os' ->> 'name') :: text AS operating_system,
+	(user_agent -> 'browser' ->> 'name') :: text AS browser,
+	(user_agent -> 'platform' ->> 'type') :: text AS platform,
+	referrer,
+	flow_direction,
+	metadata ->> 'change' as change_metadata,
+	metadata ->> 'back' as back_metadata,
+	metadata ->> 'selectedUrls' as selected_urls,
+	metadata ->> 'flag' as result_flag,
+	metadata -> 'flagSet' as result_flagset,
+	metadata -> 'displayText' ->> 'heading' as result_heading,
+	metadata -> 'displayText' ->> 'description' as result_description,
+	case
+		when has_clicked_help then metadata
+		else null
+	end as help_metadata,
+	al.user_exit as is_user_exit,
+	node_type,
+	node_title,
+	has_clicked_help,
+	input_errors,
+	CAST(
+		EXTRACT(
+			EPOCH
+			FROM
+				(al.next_log_created_at - al.created_at)
+		) as numeric (10, 1)
+	) as time_spent_on_node_seconds,
+	a.ended_at as analytics_ended_at,
+	CAST(
+		EXTRACT(
+			EPOCH
+			FROM
+				(a.ended_at - a.created_at)
+		) / 60 as numeric (10, 1)
+	) as time_spent_on_analytics_session_minutes,
+	node_id,
+	al.allow_list_answers as allow_list_answers,
+    allow_list_answer_elements->>'proposal.projectType' AS proposal_project_type,
+    allow_list_answer_elements->>'application.declaration.connection' AS application_declaration_connection,
+    allow_list_answer_elements->>'property.type' AS property_type,
+    allow_list_answer_elements->>'drawBoundary.action' AS draw_boundary_action,
+    allow_list_answer_elements->>'user.role' AS user_role,
+    allow_list_answer_elements->>'property.constraints.planning' AS property_constraints_planning
+from
+	analytics a
+	left join analytics_logs al on a.id = al.analytics_id
+	left join flows f on a.flow_id = f.id
+	left join teams t on t.id = f.team_id
+    left join lateral jsonb_array_elements(al.allow_list_answers) AS allow_list_answer_elements ON true;
+
+--  After recreating the view grant Metabase access to it
+GRANT SELECT ON public.analytics_summary TO metabase_read_only;

hasura.planx.uk/migrations/1711643574155_update_analytics_logs_allow_list_answers_array_to_object/up.sql

@@ -0,0 +1,74 @@
+-- Update the data structure of allow_list_answers to be an object rather than array of objects
+
+UPDATE public.analytics_logs
+SET allow_list_answers = allow_list_answers->0
+WHERE jsonb_typeof(allow_list_answers) = 'array' AND jsonb_array_length(allow_list_answers) = 1;
+
+-- Update the analytics_summary to handle the change to the data structure
+
+DROP VIEW public.analytics_summary;
+
+CREATE
+OR REPLACE VIEW public.analytics_summary AS
+select
+	a.id as analytics_id,
+	al.id as analytics_log_id,
+	f.slug as service_slug,
+	t.slug as team_slug,
+	a.type as analytics_type,
+	al.created_at as analytics_log_created_at,
+	a.created_at as analytics_created_at,
+	(user_agent -> 'os' ->> 'name') :: text AS operating_system,
+	(user_agent -> 'browser' ->> 'name') :: text AS browser,
+	(user_agent -> 'platform' ->> 'type') :: text AS platform,
+	referrer,
+	flow_direction,
+	metadata ->> 'change' as change_metadata,
+	metadata ->> 'back' as back_metadata,
+	metadata ->> 'selectedUrls' as selected_urls,
+	metadata ->> 'flag' as result_flag,
+	metadata -> 'flagSet' as result_flagset,
+	metadata -> 'displayText' ->> 'heading' as result_heading,
+	metadata -> 'displayText' ->> 'description' as result_description,
+	case
+		when has_clicked_help then metadata
+		else null
+	end as help_metadata,
+	al.user_exit as is_user_exit,
+	node_type,
+	node_title,
+	has_clicked_help,
+	input_errors,
+	CAST(
+		EXTRACT(
+			EPOCH
+			FROM
+				(al.next_log_created_at - al.created_at)
+		) as numeric (10, 1)
+	) as time_spent_on_node_seconds,
+	a.ended_at as analytics_ended_at,
+	CAST(
+		EXTRACT(
+			EPOCH
+			FROM
+				(a.ended_at - a.created_at)
+		) / 60 as numeric (10, 1)
+	) as time_spent_on_analytics_session_minutes,
+	node_id,
+	al.allow_list_answers as allow_list_answers,
+    al.allow_list_answers -> 'proposal.projectType' as proposal_project_type,
+    al.allow_list_answers -> 'application.declaration.connection' as application_declaration_connection,
+    al.allow_list_answers -> 'property.type' as property_type,
+    al.allow_list_answers -> 'drawBoundary.action' as draw_boundary_action,
+    al.allow_list_answers -> 'user.role' as user_role,
+    al.allow_list_answers -> 'property.constraints.planning' as property_constraints_planning
+from
+	analytics a
+	left join analytics_logs al on a.id = al.analytics_id
+	left join flows f on a.flow_id = f.id
+	left join teams t on t.id = f.team_id;
+
+--  After recreating the view grant Metabase access to it
+GRANT SELECT ON public.analytics_summary TO metabase_read_only;
+
+

hasura.planx.uk/migrations/1711645589762_alter_table_public_analytics_logs_drop_column_node_fn/down.sql

@@ -0,0 +1,3 @@
+comment on column "public"."analytics_logs"."node_fn" is E'Links to `analytics` to provide granular details about user interactions with individual questions';
+alter table "public"."analytics_logs" alter column "node_fn" drop not null;
+alter table "public"."analytics_logs" add column "node_fn" text;

hasura.planx.uk/migrations/1711645589762_alter_table_public_analytics_logs_drop_column_node_fn/up.sql

@@ -0,0 +1 @@
+alter table "public"."analytics_logs" drop column "node_fn" cascade;