Merge pull request #3362 from owid/ordinal-types

✨ add type and sort columns to variables table

db/migration/1710255240486-AddTypeAndSortToVariables.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class AddTypeAndSortToVariables1710255240486
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `ALTER TABLE variables
+                ADD COLUMN type ENUM('float', 'int', 'mixed', 'string', 'ordinal') NULL,
+                ADD COLUMN sort JSON NULL`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `ALTER TABLE variables DROP COLUMN type, DROP COLUMN sort`
+        )
+    }
+}

packages/@ourworldindata/core-table/src/CoreTableColumns.ts

@@ -588,6 +588,18 @@ class BooleanColumn extends AbstractCoreColumn<boolean> {
     }
 }
 
+class OrdinalColumn extends CategoricalColumn {
+    @imemo get allowedValuesSorted(): string[] | undefined {
+        return this.def.sort
+    }
+
+    @imemo get sortedUniqNonEmptyStringVals(): string[] {
+        return this.allowedValuesSorted
+            ? this.allowedValuesSorted
+            : super.sortedUniqNonEmptyStringVals
+    }
+}
+
 abstract class AbstractColumnWithNumberFormatting<
     T extends PrimitiveType,
 > extends AbstractCoreColumn<T> {
@@ -882,6 +894,7 @@ export const ColumnTypeMap = {
     String: StringColumn,
     SeriesAnnotation: SeriesAnnotationColumn,
     Categorical: CategoricalColumn,
+    Ordinal: OrdinalColumn,
     Region: RegionColumn,
     Continent: ContinentColumn,
     NumberOrString: NumberOrStringColumn,

packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts

@@ -8,6 +8,7 @@ import {
     OwidTableSlugs,
     OwidColumnDef,
     LegacyGrapherInterface,
+    OwidVariableDimensions,
 } from "@ourworldindata/types"
 import {
     OwidTable,
@@ -33,6 +34,7 @@ import {
     ColumnSlug,
     EPOCH_DATE,
     OwidChartDimensionInterface,
+    OwidVariableType,
 } from "@ourworldindata/utils"
 
 export const legacyToOwidTableAndDimensions = (
@@ -545,6 +547,37 @@ const fullJoinTables = (
     )
 }
 
+const variableTypeToColumnType = (type: OwidVariableType): ColumnTypeNames => {
+    switch (type) {
+        case "ordinal":
+            return ColumnTypeNames.Ordinal
+        // TODO
+        // case "string":
+        //     return ColumnTypeNames.String
+        // case "float":
+        // case "int":
+        //     return ColumnTypeNames.Numeric
+        // case "mixed":
+        default:
+            return ColumnTypeNames.NumberOrString
+    }
+}
+
+const getSortFromDimensions = (
+    dimensions: OwidVariableDimensions
+): string[] | undefined => {
+    const values = dimensions.values?.values
+    if (!values) return
+
+    const sort = values
+        .map((value) => value.name)
+        .filter((name): name is string => name !== undefined)
+
+    if (sort.length === 0) return
+
+    return sort
+}
+
 const columnDefFromOwidVariable = (
     variable: OwidVariableWithSourceAndDimension
 ): OwidColumnDef => {
@@ -574,6 +607,19 @@ const columnDefFromOwidVariable = (
     const isContinent = variable.id === 123
     const name = isContinent ? "Continent" : variable.name
 
+    // The column's type
+    const type = isContinent
+        ? ColumnTypeNames.Continent
+        : variable.type
+          ? variableTypeToColumnType(variable.type)
+          : ColumnTypeNames.NumberOrString
+
+    // Sorted values for ordinal columns
+    const sort =
+        type === ColumnTypeNames.Ordinal
+            ? getSortFromDimensions(variable.dimensions)
+            : undefined
+
     return {
         name,
         slug,
@@ -604,9 +650,8 @@ const columnDefFromOwidVariable = (
         owidVariableId: variable.id,
         owidProcessingLevel: variable.processingLevel,
         owidSchemaVersion: variable.schemaVersion,
-        type: isContinent
-            ? ColumnTypeNames.Continent
-            : ColumnTypeNames.NumberOrString,
+        type,
+        sort,
     }
 }
 

packages/@ourworldindata/types/src/OwidVariable.ts

@@ -30,6 +30,7 @@ export interface OwidVariableWithSource {
     updatePeriodDays?: number
     datasetVersion?: string
     licenses?: OwidLicense[]
+    type?: OwidVariableType
 
     // omitted:
     // code
@@ -107,6 +108,7 @@ export interface OwidVariableDimension {
 export interface OwidVariableDimensions {
     years: OwidVariableDimension
     entities: OwidVariableDimension
+    values?: OwidVariableDimension
 }
 
 export type OwidVariableDataMetadataDimensions = {
@@ -129,3 +131,5 @@ export type OwidVariableDimensionValueFull =
 export interface OwidEntityKey {
     [id: string]: OwidVariableDimensionValuePartial
 }
+
+export type OwidVariableType = "string" | "float" | "int" | "mixed" | "ordinal"

packages/@ourworldindata/types/src/dbTypes/Variables.ts

@@ -1,3 +1,4 @@
+import { OwidVariableType } from "../OwidVariable.js"
 import { OwidVariableDisplayConfigInterface } from "../OwidVariableDisplayConfigInterface.js"
 import { JsonString } from "../domainTypes/Various.js"
 import { GrapherInterface } from "../grapherTypes/GrapherTypes.js"
@@ -37,7 +38,10 @@ export interface DbInsertVariable {
     titleVariant?: string | null
     unit: string
     updatedAt?: Date | null
+    type?: OwidVariableType | null
+    sort?: JsonString | null
 }
+
 export type DbRawVariable = Required<DbInsertVariable>
 
 export interface VariableDisplayDimension {
@@ -65,6 +69,7 @@ export type DbEnrichedVariable = Omit<
     | "grapherConfigAdmin"
     | "grapherConfigETL"
     | "processingLog"
+    | "sort"
 > & {
     display: OwidVariableDisplayConfigInterface
     license: License | null
@@ -75,6 +80,7 @@ export type DbEnrichedVariable = Omit<
     grapherConfigAdmin: GrapherInterface | null
     grapherConfigETL: GrapherInterface | null
     processingLog: unknown | null
+    sort: string[] | null
 }
 
 export function parseVariableDisplayConfig(
@@ -179,6 +185,16 @@ export function serializeVariableProcessingLog(
     return processingLog ? JSON.stringify(processingLog) : null
 }
 
+export function parseVariableSort(sort: JsonString | null): string[] | null {
+    return sort ? JSON.parse(sort) : null
+}
+
+export function serializeVariableSort(
+    sort: string[] | null
+): JsonString | null {
+    return sort ? JSON.stringify(sort) : null
+}
+
 export function parseVariablesRow(row: DbRawVariable): DbEnrichedVariable {
     return {
         ...row,
@@ -193,6 +209,7 @@ export function parseVariablesRow(row: DbRawVariable): DbEnrichedVariable {
         ),
         grapherConfigETL: parseVariableGrapherConfigETL(row.grapherConfigETL),
         processingLog: parseVariableProcessingLog(row.processingLog),
+        sort: parseVariableSort(row.sort),
     }
 }
 
@@ -214,5 +231,6 @@ export function serializeVariablesRow(row: DbEnrichedVariable): DbRawVariable {
             row.grapherConfigETL
         ),
         processingLog: serializeVariableProcessingLog(row.processingLog),
+        sort: serializeVariableSort(row.sort),
     }
 }

packages/@ourworldindata/types/src/domainTypes/CoreTableTypes.ts

@@ -151,6 +151,7 @@ export enum ColumnTypeNames {
     Region = "Region",
     SeriesAnnotation = "SeriesAnnotation",
     Categorical = "Categorical",
+    Ordinal = "Ordinal",
     Continent = "Continent",
     EntityName = "EntityName",
     EntityId = "EntityId",
@@ -207,6 +208,9 @@ export interface CoreColumnDef extends ColumnColorScale {
     descriptionFromProducer?: string
     note?: string // Any internal notes the author wants to record for display in admin interfaces
 
+    // Sorted values (in case of ordinal data)
+    sort?: string[]
+
     // Color
     color?: Color // A column can have a fixed color for use in charts where the columns are series
 

packages/@ourworldindata/types/src/index.ts

@@ -370,6 +370,7 @@ export {
     type OwidProcessingLevel,
     type IndicatorTitleWithFragments,
     joinTitleFragments,
+    type OwidVariableType,
 } from "./OwidVariable.js"
 
 export type { OwidSource } from "./OwidSource.js"