feat(api): Add data breakdown support for base widgets and tests

Vizzuality · Nov 8, 2024 · 9739858 · 9739858
1 parent 30f556b
commit 9739858
Show file tree

Hide file tree

Showing 14 changed files with 330 additions and 90 deletions.
diff --git a/api/src/infrastructure/postgres-survey-answers.repository.ts b/api/src/infrastructure/postgres-survey-answers.repository.ts
@@ -2,13 +2,12 @@ import { DataSource, getMetadataArgsStorage, Repository } from 'typeorm';
 import { ISurveyAnswerRepository } from '@api/infrastructure/survey-answer-repository.interface';
 import { Inject, Logger } from '@nestjs/common';
 import { InjectDataSource } from '@nestjs/typeorm';
-import { WidgetDataFilters } from '@shared/dto/widgets/widget-data-filter';
+import { WidgetDataFilter } from '@shared/dto/widgets/widget-data-filter';
 import { SectionWithDataWidget } from '@shared/dto/sections/section.entity';
 import { SQLAdapter } from '@api/infrastructure/sql-adapter';
 import {
   BaseWidgetWithData,
   WidgetChartData,
-  WidgetData,
 } from '@shared/dto/widgets/base-widget-data.interface';
 import { WidgetUtils } from '@shared/dto/widgets/widget.utils';
 import { SurveyAnswer } from '@shared/dto/surveys/survey-answer.entity';
@@ -21,6 +20,8 @@ export class PostgresSurveyAnswerRepository
   private readonly edgeCasesMethodNameMap: Record<string, string> = {
     'total-surveys': this.addTotalSurveysDataToWidget.name,
     'total-countries': this.addTotalCountriesDataToWidget.name,
+    'adoption-of-technology-by-country':
+      this.addAdoptionOfTechnologyByCountryDataToWidget.name,
   };
 
   public constructor(
@@ -45,7 +46,7 @@ export class PostgresSurveyAnswerRepository
 
   public async addSurveyDataToSections(
     sections: SectionWithDataWidget[],
-    filters?: WidgetDataFilters,
+    filters?: WidgetDataFilter[],
   ): Promise<SectionWithDataWidget[]> {
     let filterClause: string;
     if (filters !== undefined) {
@@ -58,9 +59,7 @@ export class PostgresSurveyAnswerRepository
       const baseWidgets = section.baseWidgets;
       for (let widgetIdx = 0; widgetIdx < baseWidgets.length; widgetIdx++) {
         const widget = baseWidgets[widgetIdx];
-        widgetDataPromises.push(
-          this.appendBaseWidgetData(widget, filterClause),
-        );
+        widgetDataPromises.push(this.addDataToWidget(widget, filterClause));
       }
     }
 
@@ -70,71 +69,80 @@ export class PostgresSurveyAnswerRepository
 
   public async addSurveyDataToBaseWidget(
     widget: BaseWidgetWithData,
-    filters?: WidgetDataFilters,
+    params: { filters?: WidgetDataFilter[]; breakdown?: string } = {},
   ): Promise<BaseWidgetWithData> {
+    const { filters, breakdown } = params;
     let filterClause: string;
     if (filters !== undefined) {
       filterClause = this.sqlAdapter.generateSqlFromWidgetDataFilters(filters);
     }
-
-    await this.appendBaseWidgetData(widget, filterClause);
+    if (breakdown === undefined) {
+      await this.addDataToWidget(widget, filterClause);
+    } else {
+      await this.addBreakdownDataToWidget(widget, breakdown, filterClause);
+    }
     return widget;
   }
 
-  private async appendBaseWidgetData(
+  private async addDataToWidget(
     widget: BaseWidgetWithData,
     filterClause: string,
   ): Promise<void> {
-    const { indicator } = widget;
+    widget.data = {};
 
     // Check if the indicator is an edge case
-    const methodName = this.edgeCasesMethodNameMap[indicator];
+    const methodName = this.edgeCasesMethodNameMap[widget.indicator];
     if (methodName !== undefined) {
       return this[methodName](widget, filterClause);
     }
 
-    const widgetData: WidgetData = {};
     const [
       supportsChart,
-      supportsSingleValue,
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      supportsSingleValue, // No generic implementation for single value widgets for the time being
       supportsMap,
-      supportsNavigation,
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      supportsNavigation, //  No generic implementation navigation widgets for the time being
     ] = WidgetUtils.getSupportedVisualizations(widget);
 
+    const dataPromises = [];
     if (supportsChart === true) {
-      const totalsSql = `SELECT answer as "key", count(answer)::integer as "count", SUM(COUNT(answer)) OVER ()::integer AS total 
-FROM ${this.answersTable} ${this.sqlAdapter.appendExpressionToFilterClause(filterClause, `question_indicator = '${indicator}'`)} GROUP BY answer`;
-      const totalsResult: { key: string; count: number }[] =
-        await this.dataSource.query(totalsSql);
-
-      const arr: WidgetChartData = [];
-      for (let rowIdx = 0; rowIdx < totalsResult.length; rowIdx++) {
-        const res = totalsResult[rowIdx];
-        arr.push({ label: res.key, value: res.count, total: res.count });
-      }
-
-      widgetData.chart = arr;
+      dataPromises.push(this.addChartDataToWidget(widget, filterClause));
     }
 
-    if (supportsSingleValue === true) {
-      // TODO: Add WidgetCounterData
+    if (supportsMap === true) {
+      dataPromises.push(this.addMapDataToWidget(widget));
     }
 
-    if (supportsMap === true) {
-      const mapSql = `SELECT country_code as country, COUNT(survey_id)::integer AS "count" 
-      FROM ${this.answersTable}
-      GROUP BY country_code, question, answer
-      HAVING question = '${widget.question}' AND answer = 'Yes'`;
+    await Promise.all(dataPromises);
+  }
 
-      const result = await this.dataSource.query(mapSql);
-      widgetData.map = result;
+  private async addChartDataToWidget(
+    widget: BaseWidgetWithData,
+    filterClause: string,
+  ): Promise<void> {
+    const totalsSql = `SELECT answer as "key", count(answer)::integer as "count", SUM(COUNT(answer)) OVER ()::integer AS total 
+    FROM ${this.answersTable} ${this.sqlAdapter.appendExpressionToFilterClause(filterClause, `question_indicator = '${widget.indicator}'`)} GROUP BY answer`;
+    const totalsResult: { key: string; count: number }[] =
+      await this.dataSource.query(totalsSql);
+
+    const arr: WidgetChartData = [];
+    for (let rowIdx = 0; rowIdx < totalsResult.length; rowIdx++) {
+      const res = totalsResult[rowIdx];
+      arr.push({ label: res.key, value: res.count, total: res.count });
     }
 
-    if (supportsNavigation === true) {
-      // TODO: Add WidgetNavigationData
-    }
+    widget.data.chart = arr;
+  }
+
+  private async addMapDataToWidget(widget: BaseWidgetWithData): Promise<void> {
+    const mapSql = `SELECT country_code as country, COUNT(survey_id)::integer AS "count" 
+    FROM ${this.answersTable}
+    GROUP BY country_code, question, answer
+    HAVING question = $1 AND answer = 'Yes' ORDER BY country_code`;
 
-    widget.data = widgetData;
+    const result = await this.dataSource.query(mapSql, [widget.indicator]);
+    widget.data.map = result;
   }
 
   private async addTotalSurveysDataToWidget(
@@ -147,7 +155,7 @@ FROM ${this.answersTable} ${this.sqlAdapter.appendExpressionToFilterClause(filte
       this.dataSource.query(filteredCount),
       this.dataSource.query(totalCount),
     ]);
-    widget.data = { counter: { value, total } };
+    widget.data.counter = { value, total };
   }
 
   private async addTotalCountriesDataToWidget(
@@ -160,6 +168,67 @@ FROM ${this.answersTable} ${this.sqlAdapter.appendExpressionToFilterClause(filte
       this.dataSource.query(filteredCount),
       this.dataSource.query(totalCount),
     ]);
-    widget.data = { counter: { value, total } };
+    widget.data.counter = { value, total };
+  }
+
+  private async addAdoptionOfTechnologyByCountryDataToWidget(
+    widget: BaseWidgetWithData,
+    filterClause: string,
+  ): Promise<void> {
+    // Best workaround to reference correct question without changing the frontend title ('Adoption of technology by country' once transformed)
+    widget.indicator = 'digital-technologies-integrated';
+    await Promise.all([
+      this.addChartDataToWidget(widget, filterClause),
+      this.addMapDataToWidget(widget),
+    ]);
+    widget.indicator = 'adoption-of-technology-by-country';
+  }
+
+  private async addBreakdownDataToWidget(
+    widget: BaseWidgetWithData,
+    breakdownIndicator: string,
+    filterClause: string,
+  ): Promise<void> {
+    const sqlCode = `WITH breakdown_data AS (
+    SELECT 
+        main_answer,
+        secondary_answer,
+        COUNT(main_answer)::integer AS count,
+        SUM(COUNT(main_answer)) OVER (PARTITION BY main_answer)::integer AS total_count
+    FROM (
+        SELECT
+            main.answer AS main_answer,
+            secondary.answer AS secondary_answer
+        FROM 
+            survey_answers AS main
+        JOIN 
+            survey_answers AS secondary
+        ON 
+            main.survey_id = secondary.survey_id 
+            AND secondary.question_indicator = $1
+        ${this.sqlAdapter.appendExpressionToFilterClause(filterClause, `question_indicator = '${widget.indicator}'`, 'main')}
+    ) AS s
+    GROUP BY 
+        main_answer, secondary_answer
+    ORDER BY 
+        main_answer, secondary_answer
+)
+SELECT 
+    main_answer AS label,
+    JSON_AGG(
+        JSON_BUILD_OBJECT(
+            'label', secondary_answer,
+            'value', count,
+            'total', total_count
+        )
+    ) AS data
+FROM breakdown_data
+GROUP BY main_answer
+ORDER BY main_answer`;
+
+    const breakdown = await this.dataSource.query(sqlCode, [
+      breakdownIndicator,
+    ]);
+    widget.data = { breakdown };
   }
 }
diff --git a/api/src/infrastructure/sql-adapter.ts b/api/src/infrastructure/sql-adapter.ts
@@ -1,4 +1,4 @@
-import { WidgetDataFilters } from '@shared/dto/widgets/widget-data-filter';
+import { WidgetDataFilter } from '@shared/dto/widgets/widget-data-filter';
 import { Injectable, Logger } from '@nestjs/common';
 import { Section } from '@shared/dto/sections/section.entity';
 import { CountryISO3Map } from '@shared/constants/country-iso3.map';
@@ -42,24 +42,30 @@ export class SQLAdapter {
     return sqlCode;
   }
 
-  public generateSqlFromWidgetDataFilters(filters: WidgetDataFilters): string {
+  public generateSqlFromWidgetDataFilters(
+    filters?: WidgetDataFilter[],
+    alias?: string,
+  ): string {
+    if (filters === undefined) {
+      return '';
+    }
     let filterClause: string = 'WHERE ';
     for (const filter of filters) {
       // Countries
-      if (filter.name == 'eu-member-state') {
+      if (filter.name == 'location-country-region') {
         filterClause += '(';
         for (const filterValue of filter.values) {
-          filterClause += `country_code ${filter.operator} '${CountryISO3Map.getISO3ByCountryName(filterValue)}' OR `;
+          filterClause += `${alias === undefined ? '' : `${alias}.`}country_code ${filter.operator} '${CountryISO3Map.getISO3ByCountryName(filterValue)}' OR `;
         }
         filterClause = filterClause.slice(0, -4);
         filterClause += ') AND ';
         continue;
       }
 
-      filterClause += `(question_indicator = '${filter.name}' AND (`;
+      filterClause += `(${alias === undefined ? '' : `${alias}.`}question_indicator = '${filter.name}' AND (`;
 
       for (const filterValue of filter.values) {
-        filterClause += `answer ${filter.operator} '${filterValue}' OR `;
+        filterClause += `${alias === undefined ? '' : `${alias}.`}answer ${filter.operator} '${filterValue}' OR `;
       }
       filterClause = filterClause.slice(0, -4);
       filterClause += ')) AND ';
@@ -71,10 +77,11 @@ export class SQLAdapter {
   public appendExpressionToFilterClause(
     filterClause: string,
     newExpression: string,
+    alias?: string,
   ): string {
-    if (filterClause !== undefined) {
-      return `${filterClause} AND ${newExpression}`;
+    if (filterClause !== '') {
+      return `${filterClause} AND ${alias === undefined ? newExpression : `${alias}.${newExpression}`}`;
     }
-    return `WHERE ${newExpression}`;
+    return `WHERE  ${alias === undefined ? newExpression : `${alias}.${newExpression}`}`;
   }
 }
diff --git a/api/src/infrastructure/survey-answer-repository.interface.ts b/api/src/infrastructure/survey-answer-repository.interface.ts
@@ -17,6 +17,6 @@ export interface ISurveyAnswerRepository extends Repository<SurveyAnswer> {
   ): Promise<SectionWithDataWidget[]>;
   addSurveyDataToBaseWidget(
     widget: BaseWidget,
-    filters?: WidgetDataFilter[],
+    params: { filters?: WidgetDataFilter[]; breakdown?: string },
   ): Promise<BaseWidgetWithData>;
 }
diff --git a/api/src/modules/sections/sections.service.ts b/api/src/modules/sections/sections.service.ts
@@ -8,7 +8,7 @@ import {
   SectionWithDataWidget,
 } from '@shared/dto/sections/section.entity';
 import { FetchSpecification } from 'nestjs-base-service';
-import { WidgetDataFiltersSchema } from '@shared/schemas/widget-data-filters.schema';
+import { SearchWidgetDataFiltersSchema } from '@shared/schemas/search-widget-data-params.schema';
 import {
   ISurveyAnswerRepository,
   SurveyAnswerRepository,
@@ -31,7 +31,7 @@ export class SectionsService extends AppBaseService<
   }
 
   public async searchSectionsWithData(
-    query: FetchSpecification & WidgetDataFiltersSchema,
+    query: FetchSpecification & SearchWidgetDataFiltersSchema,
   ): Promise<SectionWithDataWidget[]> {
     // TODO: There is a bug / weird behavior in typeorm when using take and skip with leftJoinAndSelect:
     //       https://github.com/typeorm/typeorm/issues/4742#issuecomment-780702477

diff --git a/api/src/modules/widgets/widgets.service.ts b/api/src/modules/widgets/widgets.service.ts
@@ -7,7 +7,7 @@ import { Repository, SelectQueryBuilder } from 'typeorm';
 import { FetchSpecification } from 'nestjs-base-service';
 import { WidgetVisualisationFilters } from '@shared/schemas/widget-visualisation-filters.schema';
 import { BaseWidgetWithData } from '@shared/dto/widgets/base-widget-data.interface';
-import { WidgetDataFiltersSchema } from '@shared/schemas/widget-data-filters.schema';
+import { SearchWidgetDataParamsSchema } from '@shared/schemas/search-widget-data-params.schema';
 import {
   ISurveyAnswerRepository,
   SurveyAnswerRepository,
@@ -57,7 +57,7 @@ export class WidgetsService extends AppBaseService<
 
   public async findWidgetWithDataById(
     id: string,
-    query: FetchSpecification & WidgetDataFiltersSchema,
+    query: FetchSpecification & SearchWidgetDataParamsSchema,
   ): Promise<BaseWidgetWithData> {
     const widget = await this.baseWidgetRepository.findOneBy({ indicator: id });
     if (widget === null) {
@@ -71,10 +71,10 @@ export class WidgetsService extends AppBaseService<
     }
 
     const baseWidgetWithData =
-      await this.surveyAnswerRepository.addSurveyDataToBaseWidget(
-        widget,
-        query.filters,
-      );
+      await this.surveyAnswerRepository.addSurveyDataToBaseWidget(widget, {
+        filters: query.filters,
+        breakdown: query.breakdown,
+      });
 
     return baseWidgetWithData;
   }

diff --git a/api/test/e2e/sections/sections-crud.spec.ts b/api/test/e2e/sections/sections-crud.spec.ts
@@ -55,7 +55,7 @@ describe('Page Sections API', () => {
     const res = await testManager
       .request()
       .get(
-        '/sections?filters[0][name]=eu-member-state&filters[0][operator]==&filters[0][values][0]=Belgium',
+        '/sections?filters[0][name]=location-country-region&filters[0][operator]==&filters[0][values][0]=Belgium',
       ); // Implicit ?include[]=baseWidgets&sort[]=order&sort[]=baseWidget.sectionOrder
 
     // Then

diff --git a/api/test/e2e/users/custom-widgets/custom-widget-crud.spec.ts b/api/test/e2e/users/custom-widgets/custom-widget-crud.spec.ts
@@ -29,7 +29,7 @@ describe('Custom Widgets API', () => {
 
     await dataSourceManager.loadQuestionIndicatorMap();
     baseWidget = await entityMocks.createBaseWidget({
-      indicator: 'eu-member-state',
+      indicator: 'location-country-region',
     });
   });