diff --git a/adminSiteClient/admin.scss b/adminSiteClient/admin.scss
index 966b05e0a57..7fc8df085ef 100644
--- a/adminSiteClient/admin.scss
+++ b/adminSiteClient/admin.scss
@@ -1224,6 +1224,14 @@ main:not(.ChartEditorPage):not(.GdocsEditPage) {
}
.ImageIndexPage {
+ @media (max-width: 1300px) {
+ padding-left: 0 !important;
+ padding-right: 0 !important;
+ .ant-table-cell {
+ padding-left: 4px !important;
+ padding-right: 4px !important;
+ }
+ }
.ImageIndexPage__delete-user-button {
border-radius: 50%;
margin-left: 8px;
diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts
index 5cf0e042bc3..90afc087987 100644
--- a/adminSiteServer/apiRouter.ts
+++ b/adminSiteServer/apiRouter.ts
@@ -1,2896 +1,429 @@
/* eslint @typescript-eslint/no-unused-vars: [ "warn", { argsIgnorePattern: "^(res|req)$" } ] */
-import * as lodash from "lodash"
-import * as db from "../db/db.js"
-import {
- UNCATEGORIZED_TAG_ID,
- BAKE_ON_CHANGE,
- BAKED_BASE_URL,
- ADMIN_BASE_URL,
- DATA_API_URL,
- FEATURE_FLAGS,
-} from "../settings/serverSettings.js"
+import { TaggableType } from "@ourworldindata/types"
+import { DeployQueueServer } from "../baker/DeployQueueServer.js"
import {
- CLOUDFLARE_IMAGES_URL,
- FeatureFlagFeature,
-} from "../settings/clientSettings.js"
-import { expectInt, isValidSlug } from "../serverUtils/serverUtil.js"
+ updateVariableAnnotations,
+ getChartBulkUpdate,
+ updateBulkChartConfigs,
+ getVariableAnnotations,
+} from "./apiRoutes/bulkUpdates.js"
import {
- OldChartFieldList,
- assignTagsForCharts,
- getChartConfigById,
- getChartSlugById,
- getGptTopicSuggestions,
- getRedirectsByChartId,
- oldChartFieldList,
- setChartTags,
- getParentByChartConfig,
- getPatchConfigByChartId,
- isInheritanceEnabledForChart,
- getParentByChartId,
-} from "../db/model/Chart.js"
-import { Request } from "./authentication.js"
+ getChartViews,
+ getChartViewById,
+ createChartView,
+ updateChartView,
+ deleteChartView,
+} from "./apiRoutes/chartViews.js"
import {
- getMergedGrapherConfigForVariable,
- fetchS3MetadataByPath,
- fetchS3DataValuesByPath,
- searchVariables,
- getGrapherConfigsForVariable,
- updateGrapherConfigAdminOfVariable,
- updateGrapherConfigETLOfVariable,
- updateAllChartsThatInheritFromIndicator,
- updateAllMultiDimViewsThatInheritFromIndicator,
- getAllChartsForIndicator,
-} from "../db/model/Variable.js"
-import { updateExistingFullConfig } from "../db/model/ChartConfigs.js"
-import { getCanonicalUrl } from "@ourworldindata/components"
+ getDatasets,
+ getDataset,
+ updateDataset,
+ setArchived,
+ setTags,
+ deleteDataset,
+ republishCharts,
+} from "./apiRoutes/datasets.js"
+import { addExplorerTags, deleteExplorerTags } from "./apiRoutes/explorer.js"
import {
- GDOCS_BASE_URL,
- camelCaseProperties,
- GdocsContentSource,
- isEmpty,
- JsonError,
- OwidGdocPostInterface,
- parseIntOrUndefined,
- DbRawPostWithGdocPublishStatus,
- OwidVariableWithSource,
- TaggableType,
- DbChartTagJoin,
- pick,
- Json,
- checkIsGdocPostExcludingFragments,
- checkIsPlainObjectWithGuard,
- mergeGrapherConfigs,
- diffGrapherConfigs,
- omitUndefinedValues,
- getParentVariableIdFromChartConfig,
- omit,
- gdocUrlRegex,
-} from "@ourworldindata/utils"
-import { applyPatch } from "../adminShared/patchHelper.js"
+ getAllGdocIndexItems,
+ getIndividualGdoc,
+ createOrUpdateGdoc,
+ deleteGdoc,
+ setGdocTags,
+} from "./apiRoutes/gdocs.js"
import {
- OperationContext,
- parseToOperation,
-} from "../adminShared/SqlFilterSExpression.js"
+ getImagesHandler,
+ postImageHandler,
+ putImageHandler,
+ patchImageHandler,
+ deleteImageHandler,
+ getImageUsageHandler,
+} from "./apiRoutes/images.js"
+import { handleMultiDimDataPageRequest } from "./apiRoutes/mdims.js"
import {
- BulkChartEditResponseRow,
- BulkGrapherConfigResponse,
- chartBulkUpdateAllowedColumnNamesAndTypes,
- GrapherConfigPatch,
- variableAnnotationAllowedColumnNamesAndTypes,
- VariableAnnotationsResponseRow,
-} from "../adminShared/AdminSessionTypes.js"
+ fetchAllWork,
+ fetchNamespaces,
+ fetchSourceById,
+} from "./apiRoutes/misc.js"
import {
- DbPlainDatasetTag,
- GrapherInterface,
- OwidGdocType,
- DbPlainUser,
- UsersTableName,
- DbPlainTag,
- DbRawVariable,
- parseOriginsRow,
- PostsTableName,
- DbRawPost,
- DbPlainChartSlugRedirect,
- DbPlainChart,
- DbInsertChartRevision,
- serializeChartConfig,
- DbRawOrigin,
- DbRawPostGdoc,
- PostsGdocsXImagesTableName,
- PostsGdocsLinksTableName,
- PostsGdocsTableName,
- DbPlainDataset,
- DbInsertUser,
- FlatTagGraph,
- DbRawChartConfig,
- parseChartConfig,
- MultiDimDataPageConfigRaw,
- R2GrapherConfigDirectory,
- ChartConfigsTableName,
- Base64String,
- DbPlainChartView,
- ChartViewsTableName,
- DbInsertChartView,
- PostsGdocsComponentsTableName,
- CHART_VIEW_PROPS_TO_PERSIST,
- CHART_VIEW_PROPS_TO_OMIT,
- DbEnrichedImage,
- JsonString,
-} from "@ourworldindata/types"
-import { uuidv7 } from "uuidv7"
+ handleGetPostsJson,
+ handleSetTagsForPost,
+ handleGetPostById,
+ handleCreateGdoc,
+ handleUnlinkGdoc,
+} from "./apiRoutes/posts.js"
import {
- migrateGrapherConfigToLatestVersion,
- getVariableDataRoute,
- getVariableMetadataRoute,
- defaultGrapherConfig,
- grapherConfigToQueryParams,
-} from "@ourworldindata/grapher"
-import { getDatasetById, setTagsForDataset } from "../db/model/Dataset.js"
-import { getUserById, insertUser, updateUser } from "../db/model/User.js"
-import { GdocPost } from "../db/model/Gdoc/GdocPost.js"
+ handleGetSiteRedirects,
+ handlePostNewSiteRedirect,
+ handleDeleteSiteRedirect,
+ handleGetRedirects,
+ handlePostNewChartRedirect,
+ handleDeleteChartRedirect,
+} from "./apiRoutes/redirects.js"
+import { triggerStaticBuild } from "./apiRoutes/routeUtils.js"
+import { suggestGptTopics, suggestGptAltText } from "./apiRoutes/suggest.js"
import {
- syncDatasetToGitRepo,
- removeDatasetFromGitRepo,
-} from "./gitDataExport.js"
-import { denormalizeLatestCountryData } from "../baker/countryProfiles.js"
+ handleGetFlatTagGraph,
+ handlePostTagGraph,
+} from "./apiRoutes/tagGraph.js"
import {
- indexIndividualGdocPost,
- removeIndividualGdocPostFromIndex,
-} from "../baker/algolia/utils/pages.js"
-import { ChartViewMinimalInformation } from "../adminSiteClient/ChartEditor.js"
-import { DeployQueueServer } from "../baker/DeployQueueServer.js"
-import { FunctionalRouter } from "./FunctionalRouter.js"
-import Papa from "papaparse"
+ getTagById,
+ updateTag,
+ createTag,
+ getAllTags,
+ deleteTag,
+} from "./apiRoutes/tags.js"
import {
- setTagsForPost,
- getTagsByPostId,
- getWordpressPostReferencesByChartId,
- getGdocsPostReferencesByChartId,
-} from "../db/model/Post.js"
+ getUsers,
+ getUserByIdHandler,
+ deleteUser,
+ updateUserHandler,
+ addUser,
+ addImageToUser,
+ removeUserImage,
+} from "./apiRoutes/users.js"
import {
- checkHasChanges,
- checkIsLightningUpdate,
- GdocPublishingAction,
- getPublishingAction,
-} from "../adminSiteClient/gdocsDeploy.js"
-import { createGdocAndInsertOwidGdocPostContent } from "../db/model/Gdoc/archieToGdoc.js"
-import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
+ getEditorVariablesJson,
+ getVariableDataJson,
+ getVariableMetadataJson,
+ getVariablesJson,
+ getVariablesUsagesJson,
+ getVariablesGrapherConfigETLPatchConfigJson,
+ getVariablesGrapherConfigAdminPatchConfigJson,
+ getVariablesMergedGrapherConfigJson,
+ getVariablesVariableIdJson,
+ putVariablesVariableIdGrapherConfigETL,
+ deleteVariablesVariableIdGrapherConfigETL,
+ putVariablesVariableIdGrapherConfigAdmin,
+ deleteVariablesVariableIdGrapherConfigAdmin,
+ getVariablesVariableIdChartsJson,
+} from "./apiRoutes/variables.js"
+import { FunctionalRouter } from "./FunctionalRouter.js"
import {
+ patchRouteWithRWTransaction,
getRouteWithROTransaction,
- deleteRouteWithRWTransaction,
- putRouteWithRWTransaction,
postRouteWithRWTransaction,
- patchRouteWithRWTransaction,
+ putRouteWithRWTransaction,
+ deleteRouteWithRWTransaction,
getRouteNonIdempotentWithRWTransaction,
} from "./functionalRouterHelpers.js"
-import { getPublishedLinksTo } from "../db/model/Link.js"
-import {
- getChainedRedirect,
- getRedirectById,
- getRedirects,
- redirectWithSourceExists,
-} from "../db/model/Redirect.js"
-import { getMinimalGdocPostsByIds } from "../db/model/Gdoc/GdocBase.js"
import {
- GdocLinkUpdateMode,
- createOrLoadGdocById,
- gdocFromJSON,
- getAllGdocIndexItemsOrderedByUpdatedAt,
- getAndLoadGdocById,
- getGdocBaseObjectById,
- setLinksForGdoc,
- setTagsForGdoc,
- addImagesToContentGraph,
- updateGdocContentOnly,
- upsertGdoc,
-} from "../db/model/Gdoc/GdocFactory.js"
-import { match } from "ts-pattern"
-import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js"
-import { GdocHomepage } from "../db/model/Gdoc/GdocHomepage.js"
-import { GdocAbout } from "../db/model/Gdoc/GdocAbout.js"
-import { GdocAuthor } from "../db/model/Gdoc/GdocAuthor.js"
-import path from "path"
-import {
- deleteGrapherConfigFromR2,
- deleteGrapherConfigFromR2ByUUID,
- saveGrapherConfigToR2ByUUID,
-} from "./chartConfigR2Helpers.js"
-import { createMultiDimConfig } from "./multiDim.js"
-import { isMultiDimDataPagePublished } from "../db/model/MultiDimDataPage.js"
-import {
- retrieveChartConfigFromDbAndSaveToR2,
- saveNewChartConfigInDbAndR2,
- updateChartConfigInDbAndR2,
-} from "./chartConfigHelpers.js"
-import { ApiChartViewOverview } from "../adminShared/AdminTypes.js"
-import { References } from "../adminSiteClient/AbstractChartEditor.js"
-import {
- deleteFromCloudflare,
- fetchGptGeneratedAltText,
- processImageContent,
- uploadToCloudflare,
- validateImagePayload,
-} from "./imagesHelpers.js"
-import pMap from "p-map"
+ getChartsJson,
+ getChartsCsv,
+ getChartConfigJson,
+ getChartParentJson,
+ getChartPatchConfigJson,
+ getChartLogsJson,
+ getChartReferencesJson,
+ getChartRedirectsJson,
+ getChartPageviewsJson,
+ createChart,
+ setChartTagsHandler,
+ updateChart,
+ deleteChart,
+} from "./apiRoutes/charts.js"
const apiRouter = new FunctionalRouter()
-// Call this to trigger build and deployment of static charts on change
-const triggerStaticBuild = async (user: DbPlainUser, commitMessage: string) => {
- if (!BAKE_ON_CHANGE) {
- console.log(
- "Not triggering static build because BAKE_ON_CHANGE is false"
- )
- return
- }
-
- return new DeployQueueServer().enqueueChange({
- timeISOString: new Date().toISOString(),
- authorName: user.fullName,
- authorEmail: user.email,
- message: commitMessage,
- })
-}
-
-const enqueueLightningChange = async (
- user: DbPlainUser,
- commitMessage: string,
- slug: string
-) => {
- if (!BAKE_ON_CHANGE) {
- console.log(
- "Not triggering static build because BAKE_ON_CHANGE is false"
- )
- return
- }
-
- return new DeployQueueServer().enqueueChange({
- timeISOString: new Date().toISOString(),
- authorName: user.fullName,
- authorEmail: user.email,
- message: commitMessage,
- slug,
- })
-}
-
-async function getLogsByChartId(
- knex: db.KnexReadonlyTransaction,
- chartId: number
-): Promise<
- {
- userId: number
- config: Json
- userName: string
- createdAt: Date
- }[]
-> {
- const logs = await db.knexRaw<{
- userId: number
- config: string
- userName: string
- createdAt: Date
- }>(
- knex,
- `SELECT userId, config, fullName as userName, l.createdAt
- FROM chart_revisions l
- LEFT JOIN users u on u.id = userId
- WHERE chartId = ?
- ORDER BY l.id DESC
- LIMIT 50`,
- [chartId]
- )
- return logs.map((log) => ({
- ...log,
- config: JSON.parse(log.config),
- }))
-}
-
-const getReferencesByChartId = async (
- chartId: number,
- knex: db.KnexReadonlyTransaction
-): Promise
=> {
- const postsWordpressPromise = getWordpressPostReferencesByChartId(
- chartId,
- knex
- )
- const postGdocsPromise = getGdocsPostReferencesByChartId(chartId, knex)
- const explorerSlugsPromise = db.knexRaw<{ explorerSlug: string }>(
- knex,
- `SELECT DISTINCT
- explorerSlug
- FROM
- explorer_charts
- WHERE
- chartId = ?`,
- [chartId]
- )
- const chartViewsPromise = db.knexRaw(
- knex,
- `-- sql
- SELECT cv.id, cv.name, cc.full ->> "$.title" AS title
- FROM chart_views cv
- JOIN chart_configs cc ON cc.id = cv.chartConfigId
- WHERE cv.parentChartId = ?`,
- [chartId]
- )
- const [postsWordpress, postsGdocs, explorerSlugs, chartViews] =
- await Promise.all([
- postsWordpressPromise,
- postGdocsPromise,
- explorerSlugsPromise,
- chartViewsPromise,
- ])
-
- return {
- postsGdocs,
- postsWordpress,
- explorers: explorerSlugs.map(
- (row: { explorerSlug: string }) => row.explorerSlug
- ),
- chartViews,
- }
-}
-
-const expectChartById = async (
- knex: db.KnexReadonlyTransaction,
- chartId: any
-): Promise => {
- const chart = await getChartConfigById(knex, expectInt(chartId))
- if (chart) return chart.config
-
- throw new JsonError(`No chart found for id ${chartId}`, 404)
-}
-
-const expectPatchConfigByChartId = async (
- knex: db.KnexReadonlyTransaction,
- chartId: any
-): Promise => {
- const patchConfig = await getPatchConfigByChartId(knex, expectInt(chartId))
- if (!patchConfig) {
- throw new JsonError(`No chart found for id ${chartId}`, 404)
- }
- return patchConfig
-}
-
-const saveNewChart = async (
- knex: db.KnexReadWriteTransaction,
- {
- config,
- user,
- // new charts inherit by default
- shouldInherit = true,
- }: { config: GrapherInterface; user: DbPlainUser; shouldInherit?: boolean }
-): Promise<{
- chartConfigId: Base64String
- patchConfig: GrapherInterface
- fullConfig: GrapherInterface
-}> => {
- // grab the parent of the chart if inheritance should be enabled
- const parent = shouldInherit
- ? await getParentByChartConfig(knex, config)
- : undefined
-
- // compute patch and full configs
- const patchConfig = diffGrapherConfigs(config, parent?.config ?? {})
- const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig)
-
- // insert patch & full configs into the chart_configs table
- // We can't quite use `saveNewChartConfigInDbAndR2` here, because
- // we need to update the chart id in the config after inserting it.
- const chartConfigId = uuidv7() as Base64String
- await db.knexRaw(
- knex,
- `-- sql
- INSERT INTO chart_configs (id, patch, full)
- VALUES (?, ?, ?)
- `,
- [
- chartConfigId,
- serializeChartConfig(patchConfig),
- serializeChartConfig(fullConfig),
- ]
- )
-
- // add a new chart to the charts table
- const result = await db.knexRawInsert(
- knex,
- `-- sql
- INSERT INTO charts (configId, isInheritanceEnabled, lastEditedAt, lastEditedByUserId)
- VALUES (?, ?, ?, ?)
- `,
- [chartConfigId, shouldInherit, new Date(), user.id]
- )
-
- // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true
- const chartId = result.insertId
- patchConfig.id = chartId
- fullConfig.id = chartId
- await db.knexRaw(
- knex,
- `-- sql
- UPDATE chart_configs cc
- JOIN charts c ON c.configId = cc.id
- SET
- cc.patch=JSON_SET(cc.patch, '$.id', ?),
- cc.full=JSON_SET(cc.full, '$.id', ?)
- WHERE c.id = ?
- `,
- [chartId, chartId, chartId]
- )
-
- await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId)
-
- return { chartConfigId, patchConfig, fullConfig }
-}
-
-const updateExistingChart = async (
- knex: db.KnexReadWriteTransaction,
- params: {
- config: GrapherInterface
- user: DbPlainUser
- chartId: number
- // if undefined, keep inheritance as is.
- // if true or false, enable or disable inheritance
- shouldInherit?: boolean
- }
-): Promise<{
- chartConfigId: Base64String
- patchConfig: GrapherInterface
- fullConfig: GrapherInterface
-}> => {
- const { config, user, chartId } = params
-
- // make sure that the id of the incoming config matches the chart id
- config.id = chartId
-
- // if inheritance is enabled, grab the parent from its config
- const shouldInherit =
- params.shouldInherit ??
- (await isInheritanceEnabledForChart(knex, chartId))
- const parent = shouldInherit
- ? await getParentByChartConfig(knex, config)
- : undefined
-
- // compute patch and full configs
- const patchConfig = diffGrapherConfigs(config, parent?.config ?? {})
- const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig)
-
- const chartConfigIdRow = await db.knexRawFirst<
- Pick
- >(knex, `SELECT configId FROM charts WHERE id = ?`, [chartId])
-
- if (!chartConfigIdRow)
- throw new JsonError(`No chart config found for id ${chartId}`, 404)
-
- const now = new Date()
-
- const { chartConfigId } = await updateChartConfigInDbAndR2(
- knex,
- chartConfigIdRow.configId as Base64String,
- patchConfig,
- fullConfig
- )
-
- // update charts row
- await db.knexRaw(
- knex,
- `-- sql
- UPDATE charts
- SET isInheritanceEnabled=?, updatedAt=?, lastEditedAt=?, lastEditedByUserId=?
- WHERE id = ?
- `,
- [shouldInherit, now, now, user.id, chartId]
- )
-
- return { chartConfigId, patchConfig, fullConfig }
-}
-
-const saveGrapher = async (
- knex: db.KnexReadWriteTransaction,
- {
- user,
- newConfig,
- existingConfig,
- shouldInherit,
- referencedVariablesMightChange = true,
- }: {
- user: DbPlainUser
- newConfig: GrapherInterface
- existingConfig?: GrapherInterface
- // if undefined, keep inheritance as is.
- // if true or false, enable or disable inheritance
- shouldInherit?: boolean
- // if the variables a chart uses can change then we need
- // to update the latest country data which takes quite a long time (hundreds of ms)
- referencedVariablesMightChange?: boolean
- }
-) => {
- // Try to migrate the new config to the latest version
- newConfig = migrateGrapherConfigToLatestVersion(newConfig)
-
- // Slugs need some special logic to ensure public urls remain consistent whenever possible
- async function isSlugUsedInRedirect() {
- const rows = await db.knexRaw(
- knex,
- `SELECT * FROM chart_slug_redirects WHERE chart_id != ? AND slug = ?`,
- // -1 is a placeholder ID that will never exist; but we cannot use NULL because
- // in that case we would always get back an empty resultset
- [existingConfig ? existingConfig.id : -1, newConfig.slug]
- )
- return rows.length > 0
- }
-
- async function isSlugUsedInOtherGrapher() {
- const rows = await db.knexRaw>(
- knex,
- `-- sql
- SELECT c.id
- FROM charts c
- JOIN chart_configs cc ON cc.id = c.configId
- WHERE
- c.id != ?
- AND cc.full ->> "$.isPublished" = "true"
- AND cc.slug = ?
- `,
- // -1 is a placeholder ID that will never exist; but we cannot use NULL because
- // in that case we would always get back an empty resultset
- [existingConfig ? existingConfig.id : -1, newConfig.slug]
- )
- return rows.length > 0
- }
-
- // When a chart is published, check for conflicts
- if (newConfig.isPublished) {
- if (!isValidSlug(newConfig.slug))
- throw new JsonError(`Invalid chart slug ${newConfig.slug}`)
- else if (await isSlugUsedInRedirect())
- throw new JsonError(
- `This chart slug was previously used by another chart: ${newConfig.slug}`
- )
- else if (await isSlugUsedInOtherGrapher())
- throw new JsonError(
- `This chart slug is in use by another published chart: ${newConfig.slug}`
- )
- else if (
- existingConfig &&
- existingConfig.isPublished &&
- existingConfig.slug !== newConfig.slug
- ) {
- // Changing slug of an existing chart, delete any old redirect and create new one
- await db.knexRaw(
- knex,
- `DELETE FROM chart_slug_redirects WHERE chart_id = ? AND slug = ?`,
- [existingConfig.id, existingConfig.slug]
- )
- await db.knexRaw(
- knex,
- `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`,
- [existingConfig.id, existingConfig.slug]
- )
- // When we rename grapher configs, make sure to delete the old one (the new one will be saved below)
- await deleteGrapherConfigFromR2(
- R2GrapherConfigDirectory.publishedGrapherBySlug,
- `${existingConfig.slug}.json`
- )
- }
- }
-
- if (existingConfig)
- // Bump chart version, very important for cachebusting
- newConfig.version = existingConfig.version! + 1
- else if (newConfig.version)
- // If a chart is republished, we want to keep incrementing the old version number,
- // otherwise it can lead to clients receiving cached versions of the old data.
- newConfig.version += 1
- else newConfig.version = 1
-
- // add the isPublished field if is missing
- if (newConfig.isPublished === undefined) {
- newConfig.isPublished = false
- }
-
- // Execute the actual database update or creation
- let chartId: number
- let chartConfigId: Base64String
- let patchConfig: GrapherInterface
- let fullConfig: GrapherInterface
- if (existingConfig) {
- chartId = existingConfig.id!
- const configs = await updateExistingChart(knex, {
- config: newConfig,
- user,
- chartId,
- shouldInherit,
- })
- chartConfigId = configs.chartConfigId
- patchConfig = configs.patchConfig
- fullConfig = configs.fullConfig
- } else {
- const configs = await saveNewChart(knex, {
- config: newConfig,
- user,
- shouldInherit,
- })
- chartConfigId = configs.chartConfigId
- patchConfig = configs.patchConfig
- fullConfig = configs.fullConfig
- chartId = fullConfig.id!
- }
-
- // Record this change in version history
- const chartRevisionLog = {
- chartId: chartId as number,
- userId: user.id,
- config: serializeChartConfig(patchConfig),
- createdAt: new Date(),
- updatedAt: new Date(),
- } satisfies DbInsertChartRevision
- await db.knexRaw(
- knex,
- `INSERT INTO chart_revisions (chartId, userId, config, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
- [
- chartRevisionLog.chartId,
- chartRevisionLog.userId,
- chartRevisionLog.config,
- chartRevisionLog.createdAt,
- chartRevisionLog.updatedAt,
- ]
- )
-
- // Remove any old dimensions and store the new ones
- // We only note that a relationship exists between the chart and variable in the database; the actual dimension configuration is left to the json
- await db.knexRaw(knex, `DELETE FROM chart_dimensions WHERE chartId=?`, [
- chartId,
- ])
-
- const newDimensions = fullConfig.dimensions ?? []
- for (const [i, dim] of newDimensions.entries()) {
- await db.knexRaw(
- knex,
- `INSERT INTO chart_dimensions (chartId, variableId, property, \`order\`) VALUES (?, ?, ?, ?)`,
- [chartId, dim.variableId, dim.property, i]
- )
- }
-
- // So we can generate country profiles including this chart data
- if (fullConfig.isPublished && referencedVariablesMightChange)
- // TODO: remove this ad hoc knex transaction context when we switch the function to knex
- await denormalizeLatestCountryData(
- knex,
- newDimensions.map((d) => d.variableId)
- )
-
- if (fullConfig.isPublished) {
- await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId, {
- directory: R2GrapherConfigDirectory.publishedGrapherBySlug,
- filename: `${fullConfig.slug}.json`,
- })
- }
-
- if (
- fullConfig.isPublished &&
- (!existingConfig || !existingConfig.isPublished)
- ) {
- // Newly published, set publication info
- await db.knexRaw(
- knex,
- `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `,
- [new Date(), user.id, chartId]
- )
- await triggerStaticBuild(user, `Publishing chart ${fullConfig.slug}`)
- } else if (
- !fullConfig.isPublished &&
- existingConfig &&
- existingConfig.isPublished
- ) {
- // Unpublishing chart, delete any existing redirects to it
- await db.knexRaw(
- knex,
- `DELETE FROM chart_slug_redirects WHERE chart_id = ?`,
- [existingConfig.id]
- )
- await deleteGrapherConfigFromR2(
- R2GrapherConfigDirectory.publishedGrapherBySlug,
- `${existingConfig.slug}.json`
- )
- await triggerStaticBuild(user, `Unpublishing chart ${fullConfig.slug}`)
- } else if (fullConfig.isPublished)
- await triggerStaticBuild(user, `Updating chart ${fullConfig.slug}`)
-
- return {
- chartId,
- savedPatch: patchConfig,
- }
-}
-
-async function updateGrapherConfigsInR2(
- knex: db.KnexReadonlyTransaction,
- updatedCharts: { chartConfigId: string; isPublished: boolean }[],
- updatedMultiDimViews: { chartConfigId: string; isPublished: boolean }[]
-) {
- const idsToUpdate = [
- ...updatedCharts.filter(({ isPublished }) => isPublished),
- ...updatedMultiDimViews,
- ].map(({ chartConfigId }) => chartConfigId)
- const builder = knex(ChartConfigsTableName)
- .select("id", "full", "fullMd5")
- .whereIn("id", idsToUpdate)
- for await (const { id, full, fullMd5 } of builder.stream()) {
- await saveGrapherConfigToR2ByUUID(id, full, fullMd5)
- }
-}
-
-getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => {
- const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000
- const charts = await db.knexRaw(
- trx,
- `-- sql
- SELECT ${oldChartFieldList},
- round(views_365d / 365, 1) as pageviewsPerDay
- FROM charts
- JOIN chart_configs ON chart_configs.id = charts.configId
- JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
- LEFT JOIN analytics_pageviews on (analytics_pageviews.url = CONCAT("https://ourworldindata.org/grapher/", chart_configs.slug) AND chart_configs.full ->> '$.isPublished' = "true" )
- LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
- ORDER BY charts.lastEditedAt DESC LIMIT ?
- `,
- [limit]
- )
-
- await assignTagsForCharts(trx, charts)
-
- return { charts }
-})
-
-getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => {
- const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000
-
- // note: this query is extended from OldChart.listFields.
- const charts = await db.knexRaw(
- trx,
- `-- sql
- SELECT
- charts.id,
- chart_configs.full->>"$.version" AS version,
- CONCAT("${BAKED_BASE_URL}/grapher/", chart_configs.full->>"$.slug") AS url,
- CONCAT("${ADMIN_BASE_URL}", "/admin/charts/", charts.id, "/edit") AS editUrl,
- chart_configs.full->>"$.slug" AS slug,
- chart_configs.full->>"$.title" AS title,
- chart_configs.full->>"$.subtitle" AS subtitle,
- chart_configs.full->>"$.sourceDesc" AS sourceDesc,
- chart_configs.full->>"$.note" AS note,
- chart_configs.chartType AS type,
- chart_configs.full->>"$.internalNotes" AS internalNotes,
- chart_configs.full->>"$.variantName" AS variantName,
- chart_configs.full->>"$.isPublished" AS isPublished,
- chart_configs.full->>"$.tab" AS tab,
- chart_configs.chartType IS NOT NULL AS hasChartTab,
- JSON_EXTRACT(chart_configs.full, "$.hasMapTab") = true AS hasMapTab,
- chart_configs.full->>"$.originUrl" AS originUrl,
- charts.lastEditedAt,
- charts.lastEditedByUserId,
- lastEditedByUser.fullName AS lastEditedBy,
- charts.publishedAt,
- charts.publishedByUserId,
- publishedByUser.fullName AS publishedBy
- FROM charts
- JOIN chart_configs ON chart_configs.id = charts.configId
- JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
- LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
- ORDER BY charts.lastEditedAt DESC
- LIMIT ?
- `,
- [limit]
- )
- // note: retrieving references is VERY slow.
- // await Promise.all(
- // charts.map(async (chart: any) => {
- // const references = await getReferencesByChartId(chart.id)
- // chart.references = references.length
- // ? references.map((ref) => ref.url)
- // : ""
- // })
- // )
- // await Chart.assignTagsForCharts(charts)
- res.setHeader("Content-disposition", "attachment; filename=charts.csv")
- res.setHeader("content-type", "text/csv")
- const csv = Papa.unparse(charts)
- return csv
-})
-
+// Bulk chart update routes
+patchRouteWithRWTransaction(
+ apiRouter,
+ "/variable-annotations",
+ updateVariableAnnotations
+)
+getRouteWithROTransaction(apiRouter, "/chart-bulk-update", getChartBulkUpdate)
+patchRouteWithRWTransaction(
+ apiRouter,
+ "/chart-bulk-update",
+ updateBulkChartConfigs
+)
getRouteWithROTransaction(
apiRouter,
- "/charts/:chartId.config.json",
- async (req, res, trx) => expectChartById(trx, req.params.chartId)
+ "/variable-annotations",
+ getVariableAnnotations
)
+// Chart routes
+getRouteWithROTransaction(apiRouter, "/charts.json", getChartsJson)
+getRouteWithROTransaction(apiRouter, "/charts.csv", getChartsCsv)
getRouteWithROTransaction(
apiRouter,
- "/charts/:chartId.parent.json",
- async (req, res, trx) => {
- const chartId = expectInt(req.params.chartId)
- const parent = await getParentByChartId(trx, chartId)
- const isInheritanceEnabled = await isInheritanceEnabledForChart(
- trx,
- chartId
- )
- return omitUndefinedValues({
- variableId: parent?.variableId,
- config: parent?.config,
- isActive: isInheritanceEnabled,
- })
- }
+ "/charts/:chartId.config.json",
+ getChartConfigJson
)
-
getRouteWithROTransaction(
apiRouter,
- "/charts/:chartId.patchConfig.json",
- async (req, res, trx) => {
- const chartId = expectInt(req.params.chartId)
- const config = await expectPatchConfigByChartId(trx, chartId)
- return config
- }
+ "/charts/:chartId.parent.json",
+ getChartParentJson
)
-
getRouteWithROTransaction(
apiRouter,
- "/editorData/namespaces.json",
- async (req, res, trx) => {
- const rows = await db.knexRaw<{
- name: string
- description?: string
- isArchived: boolean
- }>(
- trx,
- `SELECT DISTINCT
- namespace AS name,
- namespaces.description AS description,
- namespaces.isArchived AS isArchived
- FROM active_datasets
- JOIN namespaces ON namespaces.name = active_datasets.namespace`
- )
-
- return {
- namespaces: lodash
- .sortBy(rows, (row) => row.description)
- .map((namespace) => ({
- ...namespace,
- isArchived: !!namespace.isArchived,
- })),
- }
- }
+ "/charts/:chartId.patchConfig.json",
+ getChartPatchConfigJson
)
-
getRouteWithROTransaction(
apiRouter,
"/charts/:chartId.logs.json",
- async (req, res, trx) => ({
- logs: await getLogsByChartId(
- trx,
- parseInt(req.params.chartId as string)
- ),
- })
+ getChartLogsJson
)
-
getRouteWithROTransaction(
apiRouter,
"/charts/:chartId.references.json",
- async (req, res, trx) => {
- const references = {
- references: await getReferencesByChartId(
- parseInt(req.params.chartId as string),
- trx
- ),
- }
- return references
- }
+ getChartReferencesJson
)
-
getRouteWithROTransaction(
apiRouter,
"/charts/:chartId.redirects.json",
- async (req, res, trx) => ({
- redirects: await getRedirectsByChartId(
- trx,
- parseInt(req.params.chartId as string)
- ),
- })
+ getChartRedirectsJson
)
-
getRouteWithROTransaction(
apiRouter,
"/charts/:chartId.pageviews.json",
- async (req, res, trx) => {
- const slug = await getChartSlugById(
- trx,
- parseInt(req.params.chartId as string)
- )
- if (!slug) return {}
-
- const pageviewsByUrl = await db.knexRawFirst(
- trx,
- `-- sql
- SELECT *
- FROM
- analytics_pageviews
- WHERE
- url = ?`,
- [`https://ourworldindata.org/grapher/${slug}`]
- )
-
- return {
- pageviews: pageviewsByUrl ?? undefined,
- }
- }
-)
-
-getRouteWithROTransaction(
- apiRouter,
- "/editorData/variables.json",
- async (req, res, trx) => {
- const datasets = []
- const rows = await db.knexRaw<
- Pick & {
- datasetId: number
- datasetName: string
- datasetVersion: string
- } & Pick<
- DbPlainDataset,
- "namespace" | "isPrivate" | "nonRedistributable"
- >
- >(
- trx,
- `-- sql
- SELECT
- v.name,
- v.id,
- d.id as datasetId,
- d.name as datasetName,
- d.version as datasetVersion,
- d.namespace,
- d.isPrivate,
- d.nonRedistributable
- FROM variables as v JOIN active_datasets as d ON v.datasetId = d.id
- ORDER BY d.updatedAt DESC
- `
- )
-
- let dataset:
- | {
- id: number
- name: string
- version: string
- namespace: string
- isPrivate: boolean
- nonRedistributable: boolean
- variables: { id: number; name: string }[]
- }
- | undefined
- for (const row of rows) {
- if (!dataset || row.datasetName !== dataset.name) {
- if (dataset) datasets.push(dataset)
-
- dataset = {
- id: row.datasetId,
- name: row.datasetName,
- version: row.datasetVersion,
- namespace: row.namespace,
- isPrivate: !!row.isPrivate,
- nonRedistributable: !!row.nonRedistributable,
- variables: [],
- }
- }
-
- dataset.variables.push({
- id: row.id,
- name: row.name ?? "",
- })
- }
-
- if (dataset) datasets.push(dataset)
-
- return { datasets: datasets }
- }
-)
-
-apiRouter.get("/data/variables/data/:variableStr.json", async (req, res) => {
- const variableStr = req.params.variableStr as string
- if (!variableStr) throw new JsonError("No variable id given")
- if (variableStr.includes("+"))
- throw new JsonError(
- "Requesting multiple variables at the same time is no longer supported"
- )
- const variableId = parseInt(variableStr)
- if (isNaN(variableId)) throw new JsonError("Invalid variable id")
- return await fetchS3DataValuesByPath(
- getVariableDataRoute(DATA_API_URL, variableId) + "?nocache"
- )
-})
-
-apiRouter.get(
- "/data/variables/metadata/:variableStr.json",
- async (req, res) => {
- const variableStr = req.params.variableStr as string
- if (!variableStr) throw new JsonError("No variable id given")
- if (variableStr.includes("+"))
- throw new JsonError(
- "Requesting multiple variables at the same time is no longer supported"
- )
- const variableId = parseInt(variableStr)
- if (isNaN(variableId)) throw new JsonError("Invalid variable id")
- return await fetchS3MetadataByPath(
- getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache"
- )
- }
+ getChartPageviewsJson
)
-
-postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => {
- let shouldInherit: boolean | undefined
- if (req.query.inheritance) {
- shouldInherit = req.query.inheritance === "enable"
- }
-
- try {
- const { chartId } = await saveGrapher(trx, {
- user: res.locals.user,
- newConfig: req.body,
- shouldInherit,
- })
-
- return { success: true, chartId: chartId }
- } catch (err) {
- return { success: false, error: String(err) }
- }
-})
-
+postRouteWithRWTransaction(apiRouter, "/charts", createChart)
postRouteWithRWTransaction(
apiRouter,
"/charts/:chartId/setTags",
- async (req, res, trx) => {
- const chartId = expectInt(req.params.chartId)
-
- await setChartTags(trx, chartId, req.body.tags)
-
- return { success: true }
- }
+ setChartTagsHandler
+)
+putRouteWithRWTransaction(apiRouter, "/charts/:chartId", updateChart)
+deleteRouteWithRWTransaction(apiRouter, "/charts/:chartId", deleteChart)
+
+// Chart view routes
+getRouteWithROTransaction(apiRouter, "/chartViews", getChartViews)
+getRouteWithROTransaction(apiRouter, "/chartViews/:id", getChartViewById)
+postRouteWithRWTransaction(apiRouter, "/chartViews", createChartView)
+putRouteWithRWTransaction(apiRouter, "/chartViews/:id", updateChartView)
+deleteRouteWithRWTransaction(apiRouter, "/chartViews/:id", deleteChartView)
+
+// Dataset routes
+getRouteWithROTransaction(apiRouter, "/datasets.json", getDatasets)
+getRouteWithROTransaction(apiRouter, "/datasets/:datasetId.json", getDataset)
+putRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", updateDataset)
+postRouteWithRWTransaction(
+ apiRouter,
+ "/datasets/:datasetId/setArchived",
+ setArchived
)
-
-putRouteWithRWTransaction(
+postRouteWithRWTransaction(apiRouter, "/datasets/:datasetId/setTags", setTags)
+deleteRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", deleteDataset)
+postRouteWithRWTransaction(
apiRouter,
- "/charts/:chartId",
- async (req, res, trx) => {
- let shouldInherit: boolean | undefined
- if (req.query.inheritance) {
- shouldInherit = req.query.inheritance === "enable"
- }
-
- const existingConfig = await expectChartById(trx, req.params.chartId)
-
- try {
- const { chartId, savedPatch } = await saveGrapher(trx, {
- user: res.locals.user,
- newConfig: req.body,
- existingConfig,
- shouldInherit,
- })
-
- const logs = await getLogsByChartId(
- trx,
- existingConfig.id as number
- )
- return {
- success: true,
- chartId,
- savedPatch,
- newLog: logs[0],
- }
- } catch (err) {
- return {
- success: false,
- error: String(err),
- }
- }
- }
+ "/datasets/:datasetId/charts",
+ republishCharts
)
+// explorer routes
+postRouteWithRWTransaction(apiRouter, "/explorer/:slug/tags", addExplorerTags)
deleteRouteWithRWTransaction(
apiRouter,
- "/charts/:chartId",
- async (req, res, trx) => {
- const chart = await expectChartById(trx, req.params.chartId)
- if (chart.slug) {
- const links = await getPublishedLinksTo(trx, [chart.slug])
- if (links.length) {
- const sources = links.map((link) => link.sourceSlug).join(", ")
- throw new Error(
- `Cannot delete chart in-use in the following published documents: ${sources}`
- )
- }
- }
-
- await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [
- chart.id,
- ])
- await db.knexRaw(
- trx,
- `DELETE FROM chart_slug_redirects WHERE chart_id=?`,
- [chart.id]
- )
-
- const row = await db.knexRawFirst>(
- trx,
- `SELECT configId FROM charts WHERE id = ?`,
- [chart.id]
- )
- if (!row || !row.configId)
- throw new JsonError(`No chart config found for id ${chart.id}`, 404)
- if (row) {
- await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id])
- await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [
- row.configId,
- ])
- }
-
- if (chart.isPublished)
- await triggerStaticBuild(
- res.locals.user,
- `Deleting chart ${chart.slug}`
- )
+ "/explorer/:slug/tags",
+ deleteExplorerTags
+)
- await deleteGrapherConfigFromR2ByUUID(row.configId)
- if (chart.isPublished)
- await deleteGrapherConfigFromR2(
- R2GrapherConfigDirectory.publishedGrapherBySlug,
- `${chart.slug}.json`
- )
+// Gdoc routes
+getRouteWithROTransaction(apiRouter, "/gdocs", getAllGdocIndexItems)
+getRouteNonIdempotentWithRWTransaction(
+ apiRouter,
+ "/gdocs/:id",
+ getIndividualGdoc
+)
+putRouteWithRWTransaction(apiRouter, "/gdocs/:id", createOrUpdateGdoc)
+deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", deleteGdoc)
+postRouteWithRWTransaction(apiRouter, "/gdocs/:gdocId/setTags", setGdocTags)
- return { success: true }
- }
+// Images routes
+getRouteNonIdempotentWithRWTransaction(
+ apiRouter,
+ "/images.json",
+ getImagesHandler
)
+postRouteWithRWTransaction(apiRouter, "/images", postImageHandler)
+putRouteWithRWTransaction(apiRouter, "/images/:id", putImageHandler)
+// Update alt text via patch
+patchRouteWithRWTransaction(apiRouter, "/images/:id", patchImageHandler)
+deleteRouteWithRWTransaction(apiRouter, "/images/:id", deleteImageHandler)
+getRouteWithROTransaction(apiRouter, "/images/usage", getImageUsageHandler)
+// Mdim routes
putRouteWithRWTransaction(
apiRouter,
"/multi-dim/:slug",
- async (req, res, trx) => {
- const { slug } = req.params
- if (!isValidSlug(slug)) {
- throw new JsonError(`Invalid multi-dim slug ${slug}`)
- }
- const rawConfig = req.body as MultiDimDataPageConfigRaw
- const id = await createMultiDimConfig(trx, slug, rawConfig)
- if (
- FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) &&
- (await isMultiDimDataPagePublished(trx, slug))
- ) {
- await triggerStaticBuild(
- res.locals.user,
- `Publishing multidimensional chart ${slug}`
- )
- }
- return { success: true, id }
- }
+ handleMultiDimDataPageRequest
)
-getRouteWithROTransaction(apiRouter, "/users.json", async (req, res, trx) => ({
- users: await trx
- .select(
- "id" satisfies keyof DbPlainUser,
- "email" satisfies keyof DbPlainUser,
- "fullName" satisfies keyof DbPlainUser,
- "isActive" satisfies keyof DbPlainUser,
- "isSuperuser" satisfies keyof DbPlainUser,
- "createdAt" satisfies keyof DbPlainUser,
- "updatedAt" satisfies keyof DbPlainUser,
- "lastLogin" satisfies keyof DbPlainUser,
- "lastSeen" satisfies keyof DbPlainUser
- )
- .from(UsersTableName)
- .orderBy("lastSeen", "desc"),
-}))
-
+// Misc routes
+getRouteWithROTransaction(apiRouter, "/all-work", fetchAllWork)
getRouteWithROTransaction(
apiRouter,
- "/users/:userId.json",
- async (req, res, trx) => {
- const id = parseIntOrUndefined(req.params.userId)
- if (!id) throw new JsonError("No user id given")
- const user = await getUserById(trx, id)
- return { user }
- }
+ "/editorData/namespaces.json",
+ fetchNamespaces
)
+getRouteWithROTransaction(apiRouter, "/sources/:sourceId.json", fetchSourceById)
-deleteRouteWithRWTransaction(
+// Wordpress posts routes
+getRouteWithROTransaction(apiRouter, "/posts.json", handleGetPostsJson)
+postRouteWithRWTransaction(
apiRouter,
- "/users/:userId",
- async (req, res, trx) => {
- if (!res.locals.user.isSuperuser)
- throw new JsonError("Permission denied", 403)
-
- const userId = expectInt(req.params.userId)
- await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId])
-
- return { success: true }
- }
+ "/posts/:postId/setTags",
+ handleSetTagsForPost
)
-
-putRouteWithRWTransaction(
+getRouteWithROTransaction(apiRouter, "/posts/:postId.json", handleGetPostById)
+postRouteWithRWTransaction(
apiRouter,
- "/users/:userId",
- async (req, res, trx: db.KnexReadWriteTransaction) => {
- if (!res.locals.user.isSuperuser)
- throw new JsonError("Permission denied", 403)
-
- const userId = parseIntOrUndefined(req.params.userId)
- const user =
- userId !== undefined ? await getUserById(trx, userId) : null
- if (!user) throw new JsonError("No such user", 404)
-
- user.fullName = req.body.fullName
- user.isActive = req.body.isActive
-
- await updateUser(trx, userId!, pick(user, ["fullName", "isActive"]))
-
- return { success: true }
- }
+ "/posts/:postId/createGdoc",
+ handleCreateGdoc
)
-
postRouteWithRWTransaction(
apiRouter,
- "/users/add",
- async (req, res, trx: db.KnexReadWriteTransaction) => {
- if (!res.locals.user.isSuperuser)
- throw new JsonError("Permission denied", 403)
-
- const { email, fullName } = req.body
-
- await insertUser(trx, {
- email,
- fullName,
- })
-
- return { success: true }
- }
+ "/posts/:postId/unlinkGdoc",
+ handleUnlinkGdoc
)
+// Redirects routes
+getRouteWithROTransaction(
+ apiRouter,
+ "/site-redirects.json",
+ handleGetSiteRedirects
+)
postRouteWithRWTransaction(
apiRouter,
- "/users/:userId/images/:imageId",
- async (req, res, trx) => {
- const userId = expectInt(req.params.userId)
- const imageId = expectInt(req.params.imageId)
- await trx("images").where({ id: imageId }).update({ userId })
- return { success: true }
- }
+ "/site-redirects/new",
+ handlePostNewSiteRedirect
)
-
deleteRouteWithRWTransaction(
apiRouter,
- "/users/:userId/images/:imageId",
- async (req, res, trx) => {
- const userId = expectInt(req.params.userId)
- const imageId = expectInt(req.params.imageId)
- await trx("images")
- .where({ id: imageId, userId })
- .update({ userId: null })
- return { success: true }
- }
+ "/site-redirects/:id",
+ handleDeleteSiteRedirect
+)
+getRouteWithROTransaction(apiRouter, "/redirects.json", handleGetRedirects)
+postRouteWithRWTransaction(
+ apiRouter,
+ "/charts/:chartId/redirects/new",
+ handlePostNewChartRedirect
+)
+deleteRouteWithRWTransaction(
+ apiRouter,
+ "/redirects/:id",
+ handleDeleteChartRedirect
)
+// GPT routes
getRouteWithROTransaction(
apiRouter,
- "/variables.json",
- async (req, res, trx) => {
- const limit = parseIntOrUndefined(req.query.limit as string) ?? 50
- const query = req.query.search as string
- return await searchVariables(query, limit, trx)
- }
+ `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`,
+ suggestGptTopics
)
-
getRouteWithROTransaction(
apiRouter,
- "/chart-bulk-update",
- async (
- req,
- res,
- trx
- ): Promise> => {
- const context: OperationContext = {
- grapherConfigFieldName: "chart_configs.full",
- whitelistedColumnNamesAndTypes:
- chartBulkUpdateAllowedColumnNamesAndTypes,
- }
- const filterSExpr =
- req.query.filter !== undefined
- ? parseToOperation(req.query.filter as string, context)
- : undefined
-
- const offset = parseIntOrUndefined(req.query.offset as string) ?? 0
-
- // Note that our DSL generates sql here that we splice directly into the SQL as text
- // This is a potential for a SQL injection attack but we control the DSL and are
- // careful there to only allow carefully guarded vocabularies from being used, not
- // arbitrary user input
- const whereClause = filterSExpr?.toSql() ?? "true"
- const resultsWithStringGrapherConfigs = await db.knexRaw(
- trx,
- `-- sql
- SELECT
- charts.id as id,
- chart_configs.full as config,
- charts.createdAt as createdAt,
- charts.updatedAt as updatedAt,
- charts.lastEditedAt as lastEditedAt,
- charts.publishedAt as publishedAt,
- lastEditedByUser.fullName as lastEditedByUser,
- publishedByUser.fullName as publishedByUser
- FROM charts
- LEFT JOIN chart_configs ON chart_configs.id = charts.configId
- LEFT JOIN users lastEditedByUser ON lastEditedByUser.id=charts.lastEditedByUserId
- LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId
- WHERE ${whereClause}
- ORDER BY charts.id DESC
- LIMIT 50
- OFFSET ${offset.toString()}
- `
- )
-
- const results = resultsWithStringGrapherConfigs.map((row: any) => ({
- ...row,
- config: lodash.isNil(row.config) ? null : JSON.parse(row.config),
- }))
- const resultCount = await db.knexRaw<{ count: number }>(
- trx,
- `-- sql
- SELECT count(*) as count
- FROM charts
- JOIN chart_configs ON chart_configs.id = charts.configId
- WHERE ${whereClause}
- `
- )
- return { rows: results, numTotalRows: resultCount[0].count }
- }
+ `/gpt/suggest-alt-text/:imageId`,
+ suggestGptAltText
)
-patchRouteWithRWTransaction(
+// Tag graph routes
+getRouteWithROTransaction(
apiRouter,
- "/chart-bulk-update",
- async (req, res, trx) => {
- const patchesList = req.body as GrapherConfigPatch[]
- const chartIds = new Set(patchesList.map((patch) => patch.id))
-
- const configsAndIds = await db.knexRaw<
- Pick & { config: DbRawChartConfig["full"] }
- >(
- trx,
- `-- sql
- SELECT c.id, cc.full as config
- FROM charts c
- JOIN chart_configs cc ON cc.id = c.configId
- WHERE c.id IN (?)
- `,
- [[...chartIds.values()]]
- )
- const configMap = new Map(
- configsAndIds.map((item: any) => [
- item.id,
- // make sure that the id is set, otherwise the update behaviour is weird
- // TODO: discuss if this has unintended side effects
- item.config ? { ...JSON.parse(item.config), id: item.id } : {},
- ])
- )
- const oldValuesConfigMap = new Map(configMap)
- // console.log("ids", configsAndIds.map((item : any) => item.id))
- for (const patchSet of patchesList) {
- const config = configMap.get(patchSet.id)
- configMap.set(patchSet.id, applyPatch(patchSet, config))
- }
-
- for (const [id, newConfig] of configMap.entries()) {
- await saveGrapher(trx, {
- user: res.locals.user,
- newConfig,
- existingConfig: oldValuesConfigMap.get(id),
- referencedVariablesMightChange: false,
- })
- }
-
- return { success: true }
- }
+ "/flatTagGraph.json",
+ handleGetFlatTagGraph
+)
+postRouteWithRWTransaction(apiRouter, "/tagGraph", handlePostTagGraph)
+getRouteWithROTransaction(apiRouter, "/tags/:tagId.json", getTagById)
+putRouteWithRWTransaction(apiRouter, "/tags/:tagId", updateTag)
+postRouteWithRWTransaction(apiRouter, "/tags/new", createTag)
+getRouteWithROTransaction(apiRouter, "/tags.json", getAllTags)
+deleteRouteWithRWTransaction(apiRouter, "/tags/:tagId/delete", deleteTag)
+
+// User routes
+getRouteWithROTransaction(apiRouter, "/users.json", getUsers)
+getRouteWithROTransaction(apiRouter, "/users/:userId.json", getUserByIdHandler)
+deleteRouteWithRWTransaction(apiRouter, "/users/:userId", deleteUser)
+putRouteWithRWTransaction(apiRouter, "/users/:userId", updateUserHandler)
+postRouteWithRWTransaction(apiRouter, "/users/add", addUser)
+postRouteWithRWTransaction(
+ apiRouter,
+ "/users/:userId/images/:imageId",
+ addImageToUser
+)
+deleteRouteWithRWTransaction(
+ apiRouter,
+ "/users/:userId/images/:imageId",
+ removeUserImage
)
+// Variable routes
getRouteWithROTransaction(
apiRouter,
- "/variable-annotations",
- async (
- req,
- res,
- trx
- ): Promise> => {
- const context: OperationContext = {
- grapherConfigFieldName: "grapherConfigAdmin",
- whitelistedColumnNamesAndTypes:
- variableAnnotationAllowedColumnNamesAndTypes,
- }
- const filterSExpr =
- req.query.filter !== undefined
- ? parseToOperation(req.query.filter as string, context)
- : undefined
-
- const offset = parseIntOrUndefined(req.query.offset as string) ?? 0
-
- // Note that our DSL generates sql here that we splice directly into the SQL as text
- // This is a potential for a SQL injection attack but we control the DSL and are
- // careful there to only allow carefully guarded vocabularies from being used, not
- // arbitrary user input
- const whereClause = filterSExpr?.toSql() ?? "true"
- const resultsWithStringGrapherConfigs = await db.knexRaw(
- trx,
- `-- sql
- SELECT
- variables.id as id,
- variables.name as name,
- chart_configs.patch as config,
- d.name as datasetname,
- namespaces.name as namespacename,
- variables.createdAt as createdAt,
- variables.updatedAt as updatedAt,
- variables.description as description
- FROM variables
- LEFT JOIN active_datasets as d on variables.datasetId = d.id
- LEFT JOIN namespaces on d.namespace = namespaces.name
- LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id
- WHERE ${whereClause}
- ORDER BY variables.id DESC
- LIMIT 50
- OFFSET ${offset.toString()}
- `
- )
-
- const results = resultsWithStringGrapherConfigs.map((row: any) => ({
- ...row,
- config: lodash.isNil(row.config) ? null : JSON.parse(row.config),
- }))
- const resultCount = await db.knexRaw<{ count: number }>(
- trx,
- `-- sql
- SELECT count(*) as count
- FROM variables
- LEFT JOIN active_datasets as d on variables.datasetId = d.id
- LEFT JOIN namespaces on d.namespace = namespaces.name
- LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id
- WHERE ${whereClause}
- `
- )
- return { rows: results, numTotalRows: resultCount[0].count }
- }
+ "/editorData/variables.json",
+ getEditorVariablesJson
)
-
-patchRouteWithRWTransaction(
+getRouteWithROTransaction(
apiRouter,
- "/variable-annotations",
- async (req, res, trx) => {
- const patchesList = req.body as GrapherConfigPatch[]
- const variableIds = new Set(patchesList.map((patch) => patch.id))
-
- const configsAndIds = await db.knexRaw<
- Pick & {
- grapherConfigAdmin: DbRawChartConfig["patch"]
- }
- >(
- trx,
- `-- sql
- SELECT v.id, cc.patch AS grapherConfigAdmin
- FROM variables v
- LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id
- WHERE v.id IN (?)
- `,
- [[...variableIds.values()]]
- )
- const configMap = new Map(
- configsAndIds.map((item: any) => [
- item.id,
- item.grapherConfigAdmin
- ? JSON.parse(item.grapherConfigAdmin)
- : {},
- ])
- )
- // console.log("ids", configsAndIds.map((item : any) => item.id))
- for (const patchSet of patchesList) {
- const config = configMap.get(patchSet.id)
- configMap.set(patchSet.id, applyPatch(patchSet, config))
- }
-
- for (const [variableId, newConfig] of configMap.entries()) {
- const variable = await getGrapherConfigsForVariable(trx, variableId)
- if (!variable) continue
- await updateGrapherConfigAdminOfVariable(trx, variable, newConfig)
- }
-
- return { success: true }
- }
+ "/data/variables/data/:variableStr.json",
+ getVariableDataJson
)
-
+getRouteWithROTransaction(
+ apiRouter,
+ "/data/variables/metadata/:variableStr.json",
+ getVariableMetadataJson
+)
+getRouteWithROTransaction(apiRouter, "/variables.json", getVariablesJson)
getRouteWithROTransaction(
apiRouter,
"/variables.usages.json",
- async (req, res, trx) => {
- const query = `-- sql
- SELECT
- variableId,
- COUNT(DISTINCT chartId) AS usageCount
- FROM
- chart_dimensions
- GROUP BY
- variableId
- ORDER BY
- usageCount DESC`
-
- const rows = await db.knexRaw(trx, query)
-
- return rows
- }
+ getVariablesUsagesJson
)
-
getRouteWithROTransaction(
apiRouter,
"/variables/grapherConfigETL/:variableId.patchConfig.json",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
- const variable = await getGrapherConfigsForVariable(trx, variableId)
- if (!variable) {
- throw new JsonError(`Variable with id ${variableId} not found`, 500)
- }
- return variable.etl?.patchConfig ?? {}
- }
+ getVariablesGrapherConfigETLPatchConfigJson
)
-
getRouteWithROTransaction(
apiRouter,
"/variables/grapherConfigAdmin/:variableId.patchConfig.json",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
- const variable = await getGrapherConfigsForVariable(trx, variableId)
- if (!variable) {
- throw new JsonError(`Variable with id ${variableId} not found`, 500)
- }
- return variable.admin?.patchConfig ?? {}
- }
+ getVariablesGrapherConfigAdminPatchConfigJson
)
-
getRouteWithROTransaction(
apiRouter,
"/variables/mergedGrapherConfig/:variableId.json",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
- const config = await getMergedGrapherConfigForVariable(trx, variableId)
- return config ?? {}
- }
+ getVariablesMergedGrapherConfigJson
)
-
// Used in VariableEditPage
getRouteWithROTransaction(
apiRouter,
"/variables/:variableId.json",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
-
- const variable = await fetchS3MetadataByPath(
- getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache"
- )
-
- // XXX: Patch shortName onto the end of catalogPath when it's missing,
- // a temporary hack since our S3 metadata is out of date with our DB.
- // See: https://github.com/owid/etl/issues/2135
- if (variable.catalogPath && !variable.catalogPath.includes("#")) {
- variable.catalogPath += `#${variable.shortName}`
- }
-
- const rawCharts = await db.knexRaw<
- OldChartFieldList & {
- isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"]
- config: DbRawChartConfig["full"]
- }
- >(
- trx,
- `-- sql
- SELECT ${oldChartFieldList}, charts.isInheritanceEnabled, chart_configs.full AS config
- FROM charts
- JOIN chart_configs ON chart_configs.id = charts.configId
- JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
- LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
- JOIN chart_dimensions cd ON cd.chartId = charts.id
- WHERE cd.variableId = ?
- GROUP BY charts.id
- `,
- [variableId]
- )
-
- // check for parent indicators
- const charts = rawCharts.map((chart) => {
- const parentIndicatorId = getParentVariableIdFromChartConfig(
- parseChartConfig(chart.config)
- )
- const hasParentIndicator = parentIndicatorId !== undefined
- return omit({ ...chart, hasParentIndicator }, "config")
- })
-
- await assignTagsForCharts(trx, charts)
-
- const variableWithConfigs = await getGrapherConfigsForVariable(
- trx,
- variableId
- )
- const grapherConfigETL = variableWithConfigs?.etl?.patchConfig
- const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig
- const mergedGrapherConfig =
- variableWithConfigs?.admin?.fullConfig ??
- variableWithConfigs?.etl?.fullConfig
-
- // add the variable's display field to the merged grapher config
- if (mergedGrapherConfig) {
- const [varDims, otherDims] = lodash.partition(
- mergedGrapherConfig.dimensions ?? [],
- (dim) => dim.variableId === variableId
- )
- const varDimsWithDisplay = varDims.map((dim) => ({
- display: variable.display,
- ...dim,
- }))
- mergedGrapherConfig.dimensions = [
- ...varDimsWithDisplay,
- ...otherDims,
- ]
- }
-
- const variableWithCharts: OwidVariableWithSource & {
- charts: Record
- grapherConfig: GrapherInterface | undefined
- grapherConfigETL: GrapherInterface | undefined
- grapherConfigAdmin: GrapherInterface | undefined
- } = {
- ...variable,
- charts,
- grapherConfig: mergedGrapherConfig,
- grapherConfigETL,
- grapherConfigAdmin,
- }
-
- return {
- variable: variableWithCharts,
- } /*, vardata: await getVariableData([variableId]) }*/
- }
+ getVariablesVariableIdJson
)
-
// inserts a new config or updates an existing one
putRouteWithRWTransaction(
apiRouter,
"/variables/:variableId/grapherConfigETL",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
-
- let validConfig: GrapherInterface
- try {
- validConfig = migrateGrapherConfigToLatestVersion(req.body)
- } catch (err) {
- return {
- success: false,
- error: String(err),
- }
- }
-
- const variable = await getGrapherConfigsForVariable(trx, variableId)
- if (!variable) {
- throw new JsonError(`Variable with id ${variableId} not found`, 500)
- }
-
- const { savedPatch, updatedCharts, updatedMultiDimViews } =
- await updateGrapherConfigETLOfVariable(trx, variable, validConfig)
-
- await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews)
- const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews]
-
- if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) {
- await triggerStaticBuild(
- res.locals.user,
- `Updating ETL config for variable ${variableId}`
- )
- }
-
- return { success: true, savedPatch }
- }
+ putVariablesVariableIdGrapherConfigETL
)
-
deleteRouteWithRWTransaction(
apiRouter,
"/variables/:variableId/grapherConfigETL",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
-
- const variable = await getGrapherConfigsForVariable(trx, variableId)
- if (!variable) {
- throw new JsonError(`Variable with id ${variableId} not found`, 500)
- }
-
- // no-op if the variable doesn't have an ETL config
- if (!variable.etl) return { success: true }
-
- const now = new Date()
-
- // remove reference in the variables table
- await db.knexRaw(
- trx,
- `-- sql
- UPDATE variables
- SET grapherConfigIdETL = NULL
- WHERE id = ?
- `,
- [variableId]
- )
-
- // delete row in the chart_configs table
- await db.knexRaw(
- trx,
- `-- sql
- DELETE FROM chart_configs
- WHERE id = ?
- `,
- [variable.etl.configId]
- )
-
- // update admin config if there is one
- if (variable.admin) {
- await updateExistingFullConfig(trx, {
- configId: variable.admin.configId,
- config: variable.admin.patchConfig,
- updatedAt: now,
- })
- }
-
- const updates = {
- patchConfigAdmin: variable.admin?.patchConfig,
- updatedAt: now,
- }
- const updatedCharts = await updateAllChartsThatInheritFromIndicator(
- trx,
- variableId,
- updates
- )
- const updatedMultiDimViews =
- await updateAllMultiDimViewsThatInheritFromIndicator(
- trx,
- variableId,
- updates
- )
- await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews)
- const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews]
-
- if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) {
- await triggerStaticBuild(
- res.locals.user,
- `Updating ETL config for variable ${variableId}`
- )
- }
-
- return { success: true }
- }
+ deleteVariablesVariableIdGrapherConfigETL
)
-
// inserts a new config or updates an existing one
putRouteWithRWTransaction(
apiRouter,
"/variables/:variableId/grapherConfigAdmin",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
-
- let validConfig: GrapherInterface
- try {
- validConfig = migrateGrapherConfigToLatestVersion(req.body)
- } catch (err) {
- return {
- success: false,
- error: String(err),
- }
- }
-
- const variable = await getGrapherConfigsForVariable(trx, variableId)
- if (!variable) {
- throw new JsonError(`Variable with id ${variableId} not found`, 500)
- }
-
- const { savedPatch, updatedCharts, updatedMultiDimViews } =
- await updateGrapherConfigAdminOfVariable(trx, variable, validConfig)
-
- await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews)
- const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews]
-
- if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) {
- await triggerStaticBuild(
- res.locals.user,
- `Updating admin-authored config for variable ${variableId}`
- )
- }
-
- return { success: true, savedPatch }
- }
+ putVariablesVariableIdGrapherConfigAdmin
)
-
deleteRouteWithRWTransaction(
apiRouter,
"/variables/:variableId/grapherConfigAdmin",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
-
- const variable = await getGrapherConfigsForVariable(trx, variableId)
- if (!variable) {
- throw new JsonError(`Variable with id ${variableId} not found`, 500)
- }
-
- // no-op if the variable doesn't have an admin-authored config
- if (!variable.admin) return { success: true }
-
- const now = new Date()
-
- // remove reference in the variables table
- await db.knexRaw(
- trx,
- `-- sql
- UPDATE variables
- SET grapherConfigIdAdmin = NULL
- WHERE id = ?
- `,
- [variableId]
- )
-
- // delete row in the chart_configs table
- await db.knexRaw(
- trx,
- `-- sql
- DELETE FROM chart_configs
- WHERE id = ?
- `,
- [variable.admin.configId]
- )
-
- const updates = {
- patchConfigETL: variable.etl?.patchConfig,
- updatedAt: now,
- }
- const updatedCharts = await updateAllChartsThatInheritFromIndicator(
- trx,
- variableId,
- updates
- )
- const updatedMultiDimViews =
- await updateAllMultiDimViewsThatInheritFromIndicator(
- trx,
- variableId,
- updates
- )
- await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews)
- const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews]
-
- if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) {
- await triggerStaticBuild(
- res.locals.user,
- `Updating admin-authored config for variable ${variableId}`
- )
- }
-
- return { success: true }
- }
-)
-
-getRouteWithROTransaction(
- apiRouter,
- "/variables/:variableId/charts.json",
- async (req, res, trx) => {
- const variableId = expectInt(req.params.variableId)
- const charts = await getAllChartsForIndicator(trx, variableId)
- return charts.map((chart) => ({
- id: chart.chartId,
- title: chart.config.title,
- variantName: chart.config.variantName,
- isChild: chart.isChild,
- isInheritanceEnabled: chart.isInheritanceEnabled,
- isPublished: chart.isPublished,
- }))
- }
-)
-
-getRouteWithROTransaction(
- apiRouter,
- "/datasets.json",
- async (req, res, trx) => {
- const datasets = await db.knexRaw>(
- trx,
- `-- sql
- WITH variable_counts AS (
- SELECT
- v.datasetId,
- COUNT(DISTINCT cd.chartId) as numCharts
- FROM chart_dimensions cd
- JOIN variables v ON cd.variableId = v.id
- GROUP BY v.datasetId
- )
- SELECT
- ad.id,
- ad.namespace,
- ad.name,
- d.shortName,
- ad.description,
- ad.dataEditedAt,
- du.fullName AS dataEditedByUserName,
- ad.metadataEditedAt,
- mu.fullName AS metadataEditedByUserName,
- ad.isPrivate,
- ad.nonRedistributable,
- d.version,
- vc.numCharts
- FROM active_datasets ad
- LEFT JOIN variable_counts vc ON ad.id = vc.datasetId
- JOIN users du ON du.id=ad.dataEditedByUserId
- JOIN users mu ON mu.id=ad.metadataEditedByUserId
- JOIN datasets d ON d.id=ad.id
- ORDER BY ad.dataEditedAt DESC
- `
- )
-
- const tags = await db.knexRaw<
- Pick &
- Pick
- >(
- trx,
- `-- sql
- SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt
- JOIN tags t ON dt.tagId = t.id
- `
- )
- const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId)
- for (const dataset of datasets) {
- dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) =>
- lodash.omit(t, "datasetId")
- )
- }
- /*LEFT JOIN variables AS v ON v.datasetId=d.id
- GROUP BY d.id*/
-
- return { datasets: datasets }
- }
-)
-
-getRouteWithROTransaction(
- apiRouter,
- "/datasets/:datasetId.json",
- async (req: Request, res, trx) => {
- const datasetId = expectInt(req.params.datasetId)
-
- const dataset = await db.knexRawFirst>(
- trx,
- `-- sql
- SELECT d.id,
- d.namespace,
- d.name,
- d.shortName,
- d.version,
- d.description,
- d.updatedAt,
- d.dataEditedAt,
- d.dataEditedByUserId,
- du.fullName AS dataEditedByUserName,
- d.metadataEditedAt,
- d.metadataEditedByUserId,
- mu.fullName AS metadataEditedByUserName,
- d.isPrivate,
- d.isArchived,
- d.nonRedistributable,
- d.updatePeriodDays
- FROM datasets AS d
- JOIN users du ON du.id=d.dataEditedByUserId
- JOIN users mu ON mu.id=d.metadataEditedByUserId
- WHERE d.id = ?
- `,
- [datasetId]
- )
-
- if (!dataset)
- throw new JsonError(`No dataset by id '${datasetId}'`, 404)
-
- const zipFile = await db.knexRawFirst<{ filename: string }>(
- trx,
- `SELECT filename FROM dataset_files WHERE datasetId=?`,
- [datasetId]
- )
- if (zipFile) dataset.zipFile = zipFile
-
- const variables = await db.knexRaw<
- Pick<
- DbRawVariable,
- "id" | "name" | "description" | "display" | "catalogPath"
- >
- >(
- trx,
- `-- sql
- SELECT
- v.id,
- v.name,
- v.description,
- v.display,
- v.catalogPath
- FROM
- variables AS v
- WHERE
- v.datasetId = ?
- `,
- [datasetId]
- )
-
- for (const v of variables) {
- v.display = JSON.parse(v.display)
- }
-
- dataset.variables = variables
-
- // add all origins
- const origins: DbRawOrigin[] = await db.knexRaw(
- trx,
- `-- sql
- SELECT DISTINCT
- o.*
- FROM
- origins_variables AS ov
- JOIN origins AS o ON ov.originId = o.id
- JOIN variables AS v ON ov.variableId = v.id
- WHERE
- v.datasetId = ?
- `,
- [datasetId]
- )
-
- const parsedOrigins = origins.map(parseOriginsRow)
-
- dataset.origins = parsedOrigins
-
- const sources = await db.knexRaw<{
- id: number
- name: string
- description: string
- }>(
- trx,
- `
- SELECT s.id, s.name, s.description
- FROM sources AS s
- WHERE s.datasetId = ?
- ORDER BY s.id ASC
- `,
- [datasetId]
- )
-
- // expand description of sources and add to dataset as variableSources
- dataset.variableSources = sources.map((s: any) => {
- return {
- id: s.id,
- name: s.name,
- ...JSON.parse(s.description),
- }
- })
-
- const charts = await db.knexRaw(
- trx,
- `-- sql
- SELECT ${oldChartFieldList}
- FROM charts
- JOIN chart_configs ON chart_configs.id = charts.configId
- JOIN chart_dimensions AS cd ON cd.chartId = charts.id
- JOIN variables AS v ON cd.variableId = v.id
- JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
- LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
- WHERE v.datasetId = ?
- GROUP BY charts.id
- `,
- [datasetId]
- )
-
- dataset.charts = charts
-
- await assignTagsForCharts(trx, charts)
-
- const tags = await db.knexRaw<{ id: number; name: string }>(
- trx,
- `
- SELECT t.id, t.name
- FROM tags t
- JOIN dataset_tags dt ON dt.tagId = t.id
- WHERE dt.datasetId = ?
- `,
- [datasetId]
- )
- dataset.tags = tags
-
- const availableTags = await db.knexRaw<{
- id: number
- name: string
- parentName: string
- }>(
- trx,
- `
- SELECT t.id, t.name, p.name AS parentName
- FROM tags AS t
- JOIN tags AS p ON t.parentId=p.id
- `
- )
- dataset.availableTags = availableTags
-
- return { dataset: dataset }
- }
-)
-
-putRouteWithRWTransaction(
- apiRouter,
- "/datasets/:datasetId",
- async (req, res, trx) => {
- // Only updates `nonRedistributable` and `tags`, other fields come from ETL
- // and are not editable
- const datasetId = expectInt(req.params.datasetId)
- const dataset = await getDatasetById(trx, datasetId)
- if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
-
- const newDataset = (req.body as { dataset: any }).dataset
- await db.knexRaw(
- trx,
- `
- UPDATE datasets
- SET
- nonRedistributable=?,
- metadataEditedAt=?,
- metadataEditedByUserId=?
- WHERE id=?
- `,
- [
- newDataset.nonRedistributable,
- new Date(),
- res.locals.user.id,
- datasetId,
- ]
- )
-
- const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId])
- await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [
- datasetId,
- ])
- if (tagRows.length)
- for (const tagRow of tagRows) {
- await db.knexRaw(
- trx,
- `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`,
- tagRow
- )
- }
-
- try {
- await syncDatasetToGitRepo(trx, datasetId, {
- oldDatasetName: dataset.name,
- commitName: res.locals.user.fullName,
- commitEmail: res.locals.user.email,
- })
- } catch (err) {
- await logErrorAndMaybeSendToBugsnag(err, req)
- // Continue
- }
-
- return { success: true }
- }
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/datasets/:datasetId/setArchived",
- async (req, res, trx) => {
- const datasetId = expectInt(req.params.datasetId)
- const dataset = await getDatasetById(trx, datasetId)
- if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
-
- await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [
- datasetId,
- ])
- return { success: true }
- }
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/datasets/:datasetId/setTags",
- async (req, res, trx) => {
- const datasetId = expectInt(req.params.datasetId)
-
- await setTagsForDataset(trx, datasetId, req.body.tagIds)
-
- return { success: true }
- }
-)
-
-deleteRouteWithRWTransaction(
- apiRouter,
- "/datasets/:datasetId",
- async (req, res, trx) => {
- const datasetId = expectInt(req.params.datasetId)
-
- const dataset = await getDatasetById(trx, datasetId)
- if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
-
- await db.knexRaw(
- trx,
- `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`,
- [datasetId]
- )
- await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [
- datasetId,
- ])
- await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [
- datasetId,
- ])
- await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [
- datasetId,
- ])
- await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId])
-
- try {
- await removeDatasetFromGitRepo(dataset.name, dataset.namespace, {
- commitName: res.locals.user.fullName,
- commitEmail: res.locals.user.email,
- })
- } catch (err: any) {
- await logErrorAndMaybeSendToBugsnag(err, req)
- // Continue
- }
-
- return { success: true }
- }
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/datasets/:datasetId/charts",
- async (req, res, trx) => {
- const datasetId = expectInt(req.params.datasetId)
-
- const dataset = await getDatasetById(trx, datasetId)
- if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
-
- if (req.body.republish) {
- await db.knexRaw(
- trx,
- `-- sql
- UPDATE chart_configs cc
- JOIN charts c ON c.configId = cc.id
- SET
- cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1),
- cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1)
- WHERE c.id IN (
- SELECT DISTINCT chart_dimensions.chartId
- FROM chart_dimensions
- JOIN variables ON variables.id = chart_dimensions.variableId
- WHERE variables.datasetId = ?
- )`,
- [datasetId]
- )
- }
-
- await triggerStaticBuild(
- res.locals.user,
- `Republishing all charts in dataset ${dataset.name} (${dataset.id})`
- )
-
- return { success: true }
- }
-)
-
-// Get a list of redirects that map old slugs to charts
-getRouteWithROTransaction(
- apiRouter,
- "/redirects.json",
- async (req, res, trx) => ({
- redirects: await db.knexRaw(
- trx,
- `-- sql
- SELECT
- r.id,
- r.slug,
- r.chart_id as chartId,
- chart_configs.slug AS chartSlug
- FROM chart_slug_redirects AS r
- JOIN charts ON charts.id = r.chart_id
- JOIN chart_configs ON chart_configs.id = charts.configId
- ORDER BY r.id DESC
- `
- ),
- })
-)
-
-getRouteWithROTransaction(
- apiRouter,
- "/site-redirects.json",
- async (req, res, trx) => ({ redirects: await getRedirects(trx) })
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/site-redirects/new",
- async (req: Request, res, trx) => {
- const { source, target } = req.body
- const sourceAsUrl = new URL(source, "https://ourworldindata.org")
- if (sourceAsUrl.pathname === "/")
- throw new JsonError("Cannot redirect from /", 400)
- if (await redirectWithSourceExists(trx, source)) {
- throw new JsonError(
- `Redirect with source ${source} already exists`,
- 400
- )
- }
- const chainedRedirect = await getChainedRedirect(trx, source, target)
- if (chainedRedirect) {
- throw new JsonError(
- "Creating this redirect would create a chain, redirect from " +
- `${chainedRedirect.source} to ${chainedRedirect.target} ` +
- "already exists. " +
- (target === chainedRedirect.source
- ? `Please create the redirect from ${source} to ` +
- `${chainedRedirect.target} directly instead.`
- : `Please delete the existing redirect and create a ` +
- `new redirect from ${chainedRedirect.source} to ` +
- `${target} instead.`),
- 400
- )
- }
- const { insertId: id } = await db.knexRawInsert(
- trx,
- `INSERT INTO redirects (source, target) VALUES (?, ?)`,
- [source, target]
- )
- await triggerStaticBuild(
- res.locals.user,
- `Creating redirect id=${id} source=${source} target=${target}`
- )
- return { success: true, redirect: { id, source, target } }
- }
-)
-
-deleteRouteWithRWTransaction(
- apiRouter,
- "/site-redirects/:id",
- async (req, res, trx) => {
- const id = expectInt(req.params.id)
- const redirect = await getRedirectById(trx, id)
- if (!redirect) {
- throw new JsonError(`No redirect found for id ${id}`, 404)
- }
- await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id])
- await triggerStaticBuild(
- res.locals.user,
- `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}`
- )
- return { success: true }
- }
-)
-
-getRouteWithROTransaction(
- apiRouter,
- "/tags/:tagId.json",
- async (req, res, trx) => {
- const tagId = expectInt(req.params.tagId) as number | null
-
- // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff
- // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag
- // every time we create a new chart etcs
- const uncategorized = tagId === UNCATEGORIZED_TAG_ID
-
- // TODO: when we have types for our endpoints, make tag of that type instead of any
- const tag: any = await db.knexRawFirst<
- Pick<
- DbPlainTag,
- | "id"
- | "name"
- | "specialType"
- | "updatedAt"
- | "parentId"
- | "slug"
- >
- >(
- trx,
- `-- sql
- SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug
- FROM tags t LEFT JOIN tags p ON t.parentId=p.id
- WHERE t.id = ?
- `,
- [tagId]
- )
-
- // Datasets tagged with this tag
- const datasets = await db.knexRaw<
- Pick<
- DbPlainDataset,
- | "id"
- | "namespace"
- | "name"
- | "description"
- | "createdAt"
- | "updatedAt"
- | "dataEditedAt"
- | "isPrivate"
- | "nonRedistributable"
- > & { dataEditedByUserName: string }
- >(
- trx,
- `-- sql
- SELECT
- d.id,
- d.namespace,
- d.name,
- d.description,
- d.createdAt,
- d.updatedAt,
- d.dataEditedAt,
- du.fullName AS dataEditedByUserName,
- d.isPrivate,
- d.nonRedistributable
- FROM active_datasets d
- JOIN users du ON du.id=d.dataEditedByUserId
- LEFT JOIN dataset_tags dt ON dt.datasetId = d.id
- WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"}
- ORDER BY d.dataEditedAt DESC
- `,
- uncategorized ? [] : [tagId]
- )
- tag.datasets = datasets
-
- // The other tags for those datasets
- if (tag.datasets.length) {
- if (uncategorized) {
- for (const dataset of tag.datasets) dataset.tags = []
- } else {
- const datasetTags = await db.knexRaw<{
- datasetId: number
- id: number
- name: string
- }>(
- trx,
- `-- sql
- SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt
- JOIN tags t ON dt.tagId = t.id
- WHERE dt.datasetId IN (?)
- `,
- [tag.datasets.map((d: any) => d.id)]
- )
- const tagsByDatasetId = lodash.groupBy(
- datasetTags,
- (t) => t.datasetId
- )
- for (const dataset of tag.datasets) {
- dataset.tags = tagsByDatasetId[dataset.id].map((t) =>
- lodash.omit(t, "datasetId")
- )
- }
- }
- }
-
- // Charts using datasets under this tag
- const charts = await db.knexRaw(
- trx,
- `-- sql
- SELECT ${oldChartFieldList} FROM charts
- JOIN chart_configs ON chart_configs.id = charts.configId
- LEFT JOIN chart_tags ct ON ct.chartId=charts.id
- JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
- LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
- WHERE ct.tagId ${tagId === UNCATEGORIZED_TAG_ID ? "IS NULL" : "= ?"}
- GROUP BY charts.id
- ORDER BY charts.updatedAt DESC
- `,
- uncategorized ? [] : [tagId]
- )
- tag.charts = charts
-
- await assignTagsForCharts(trx, charts)
-
- // Subcategories
- const children = await db.knexRaw<{ id: number; name: string }>(
- trx,
- `-- sql
- SELECT t.id, t.name FROM tags t
- WHERE t.parentId = ?
- `,
- [tag.id]
- )
- tag.children = children
-
- // Possible parents to choose from
- const possibleParents = await db.knexRaw<{ id: number; name: string }>(
- trx,
- `-- sql
- SELECT t.id, t.name FROM tags t
- WHERE t.parentId IS NULL
- `
- )
- tag.possibleParents = possibleParents
-
- return {
- tag,
- }
- }
-)
-
-putRouteWithRWTransaction(
- apiRouter,
- "/tags/:tagId",
- async (req: Request, res, trx) => {
- const tagId = expectInt(req.params.tagId)
- const tag = (req.body as { tag: any }).tag
- await db.knexRaw(
- trx,
- `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`,
- [tag.name, new Date(), tag.slug, tagId]
- )
- if (tag.slug) {
- // See if there's a published gdoc with a matching slug.
- // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index,
- // where the page for the topic is just an article.
- const gdoc = await db.knexRaw>(
- trx,
- `-- sql
- SELECT slug FROM posts_gdocs pg
- WHERE EXISTS (
- SELECT 1
- FROM posts_gdocs_x_tags gt
- WHERE pg.id = gt.gdocId AND gt.tagId = ?
- ) AND pg.published = TRUE AND pg.slug = ?`,
- [tagId, tag.slug]
- )
- if (!gdoc.length) {
- return {
- success: true,
- tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug.
-
-Are you sure you haven't made a typo?`,
- }
- }
- }
- return { success: true }
- }
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/tags/new",
- async (req: Request, res, trx) => {
- const tag = req.body
- function validateTag(
- tag: unknown
- ): tag is { name: string; slug: string | null } {
- return (
- checkIsPlainObjectWithGuard(tag) &&
- typeof tag.name === "string" &&
- (tag.slug === null ||
- (typeof tag.slug === "string" && tag.slug !== ""))
- )
- }
- if (!validateTag(tag)) throw new JsonError("Invalid tag", 400)
-
- const conflictingTag = await db.knexRawFirst<{
- name: string
- slug: string | null
- }>(
- trx,
- `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`,
- [tag.name, tag.slug]
- )
- if (conflictingTag)
- throw new JsonError(
- conflictingTag.name === tag.name
- ? `Tag with name ${tag.name} already exists`
- : `Tag with slug ${tag.slug} already exists`,
- 400
- )
-
- const now = new Date()
- const result = await db.knexRawInsert(
- trx,
- `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`,
- // parentId will be deprecated soon once we migrate fully to the tag graph
- [tag.name, tag.slug, now, now]
- )
- return { success: true, tagId: result.insertId }
- }
-)
-
-getRouteWithROTransaction(apiRouter, "/tags.json", async (req, res, trx) => {
- return { tags: await db.getMinimalTagsWithIsTopic(trx) }
-})
-
-deleteRouteWithRWTransaction(
- apiRouter,
- "/tags/:tagId/delete",
- async (req, res, trx) => {
- const tagId = expectInt(req.params.tagId)
-
- await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId])
-
- return { success: true }
- }
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/charts/:chartId/redirects/new",
- async (req: Request, res, trx) => {
- const chartId = expectInt(req.params.chartId)
- const fields = req.body as { slug: string }
- const result = await db.knexRawInsert(
- trx,
- `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`,
- [chartId, fields.slug]
- )
- const redirectId = result.insertId
- const redirect = await db.knexRaw(
- trx,
- `SELECT * FROM chart_slug_redirects WHERE id = ?`,
- [redirectId]
- )
- return { success: true, redirect: redirect }
- }
-)
-
-deleteRouteWithRWTransaction(
- apiRouter,
- "/redirects/:id",
- async (req, res, trx) => {
- const id = expectInt(req.params.id)
-
- const redirect = await db.knexRawFirst(
- trx,
- `SELECT * FROM chart_slug_redirects WHERE id = ?`,
- [id]
- )
-
- if (!redirect)
- throw new JsonError(`No redirect found for id ${id}`, 404)
-
- await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [
- id,
- ])
- await triggerStaticBuild(
- res.locals.user,
- `Deleting redirect from ${redirect.slug}`
- )
-
- return { success: true }
- }
-)
-
-getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => {
- const raw_rows = await db.knexRaw(
- trx,
- `-- sql
- WITH
- posts_tags_aggregated AS (
- SELECT
- post_id,
- IF(
- COUNT(tags.id) = 0,
- JSON_ARRAY(),
- JSON_ARRAYAGG(JSON_OBJECT("id", tags.id, "name", tags.name))
- ) AS tags
- FROM
- post_tags
- LEFT JOIN tags ON tags.id = post_tags.tag_id
- GROUP BY
- post_id
- ),
- post_gdoc_slug_successors AS (
- SELECT
- posts.id,
- IF(
- COUNT(gdocSlugSuccessor.id) = 0,
- JSON_ARRAY(),
- JSON_ARRAYAGG(
- JSON_OBJECT("id", gdocSlugSuccessor.id, "published", gdocSlugSuccessor.published)
- )
- ) AS gdocSlugSuccessors
- FROM
- posts
- LEFT JOIN posts_gdocs gdocSlugSuccessor ON gdocSlugSuccessor.slug = posts.slug
- GROUP BY
- posts.id
- )
- SELECT
- posts.id AS id,
- posts.title AS title,
- posts.type AS TYPE,
- posts.slug AS slug,
- STATUS,
- updated_at_in_wordpress,
- posts.authors,
- posts_tags_aggregated.tags AS tags,
- gdocSuccessorId,
- gdocSuccessor.published AS isGdocSuccessorPublished,
- -- posts can either have explict successors via the gdocSuccessorId column
- -- or implicit successors if a gdoc has been created that uses the same slug
- -- as a Wp post (the gdoc one wins once it is published)
- post_gdoc_slug_successors.gdocSlugSuccessors AS gdocSlugSuccessors
- FROM
- posts
- LEFT JOIN post_gdoc_slug_successors ON post_gdoc_slug_successors.id = posts.id
- LEFT JOIN posts_gdocs gdocSuccessor ON gdocSuccessor.id = posts.gdocSuccessorId
- LEFT JOIN posts_tags_aggregated ON posts_tags_aggregated.post_id = posts.id
- ORDER BY
- updated_at_in_wordpress DESC`,
- []
- )
- const rows = raw_rows.map((row: any) => ({
- ...row,
- tags: JSON.parse(row.tags),
- isGdocSuccessorPublished: !!row.isGdocSuccessorPublished,
- gdocSlugSuccessors: JSON.parse(row.gdocSlugSuccessors),
- authors: JSON.parse(row.authors),
- }))
-
- return { posts: rows }
-})
-
-postRouteWithRWTransaction(
- apiRouter,
- "/posts/:postId/setTags",
- async (req, res, trx) => {
- const postId = expectInt(req.params.postId)
-
- await setTagsForPost(trx, postId, req.body.tagIds)
-
- return { success: true }
- }
-)
-
-getRouteWithROTransaction(
- apiRouter,
- "/posts/:postId.json",
- async (req, res, trx) => {
- const postId = expectInt(req.params.postId)
- const post = (await trx
- .table(PostsTableName)
- .where({ id: postId })
- .select("*")
- .first()) as DbRawPost | undefined
- return camelCaseProperties({ ...post })
- }
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/posts/:postId/createGdoc",
- async (req: Request, res, trx) => {
- const postId = expectInt(req.params.postId)
- const allowRecreate = !!req.body.allowRecreate
- const post = (await trx
- .table("posts_with_gdoc_publish_status")
- .where({ id: postId })
- .select("*")
- .first()) as DbRawPostWithGdocPublishStatus | undefined
-
- if (!post) throw new JsonError(`No post found for id ${postId}`, 404)
- const existingGdocId = post.gdocSuccessorId
- if (!allowRecreate && existingGdocId)
- throw new JsonError("A gdoc already exists for this post", 400)
- if (allowRecreate && existingGdocId && post.isGdocPublished) {
- throw new JsonError(
- "A gdoc already exists for this post and it is already published",
- 400
- )
- }
- if (post.archieml === null)
- throw new JsonError(
- `ArchieML was not present for post with id ${postId}`,
- 500
- )
- const tagsByPostId = await getTagsByPostId(trx)
- const tags = tagsByPostId.get(postId) || []
- const archieMl = JSON.parse(
- // Google Docs interprets ®ion in grapher URLS as ®ion
- // So we escape them here
- post.archieml.replaceAll("&", "&")
- ) as OwidGdocPostInterface
- const gdocId = await createGdocAndInsertOwidGdocPostContent(
- archieMl.content,
- post.gdocSuccessorId
- )
- // If we did not yet have a gdoc associated with this post, we need to register
- // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise
- // we don't need to make changes to the DB (only the gdoc regeneration was required)
- if (!existingGdocId) {
- post.gdocSuccessorId = gdocId
- // This is not ideal - we are using knex for on thing and typeorm for another
- // which means that we can't wrap this in a transaction. We should probably
- // move posts to use typeorm as well or at least have a typeorm alternative for it
- await trx
- .table(PostsTableName)
- .where({ id: postId })
- .update("gdocSuccessorId", gdocId)
-
- const gdoc = new GdocPost(gdocId)
- gdoc.slug = post.slug
- gdoc.content.title = post.title
- gdoc.content.type = archieMl.content.type || OwidGdocType.Article
- gdoc.published = false
- gdoc.createdAt = new Date()
- gdoc.publishedAt = post.published_at
- await upsertGdoc(trx, gdoc)
- await setTagsForGdoc(trx, gdocId, tags)
- }
- return { googleDocsId: gdocId }
- }
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/posts/:postId/unlinkGdoc",
- async (req: Request, res, trx) => {
- const postId = expectInt(req.params.postId)
- const post = (await trx
- .table("posts_with_gdoc_publish_status")
- .where({ id: postId })
- .select("*")
- .first()) as DbRawPostWithGdocPublishStatus | undefined
-
- if (!post) throw new JsonError(`No post found for id ${postId}`, 404)
- const existingGdocId = post.gdocSuccessorId
- if (!existingGdocId)
- throw new JsonError("No gdoc exists for this post", 400)
- if (existingGdocId && post.isGdocPublished) {
- throw new JsonError(
- "The GDoc is already published - you can't unlink it",
- 400
- )
- }
- // This is not ideal - we are using knex for on thing and typeorm for another
- // which means that we can't wrap this in a transaction. We should probably
- // move posts to use typeorm as well or at least have a typeorm alternative for it
- await trx
- .table(PostsTableName)
- .where({ id: postId })
- .update("gdocSuccessorId", null)
-
- await trx
- .table(PostsGdocsTableName)
- .where({ id: existingGdocId })
- .delete()
-
- return { success: true }
- }
-)
-
-getRouteWithROTransaction(
- apiRouter,
- "/sources/:sourceId.json",
- async (req: Request, res, trx) => {
- const sourceId = expectInt(req.params.sourceId)
-
- const source = await db.knexRawFirst>(
- trx,
- `
- SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace
- FROM sources AS s
- JOIN active_datasets AS d ON d.id=s.datasetId
- WHERE s.id=?`,
- [sourceId]
- )
- if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404)
- source.variables = await db.knexRaw(
- trx,
- `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`,
- [sourceId]
- )
-
- return { source: source }
- }
-)
+ deleteVariablesVariableIdGrapherConfigAdmin
+)
+getRouteWithROTransaction(
+ apiRouter,
+ "/variables/:variableId/charts.json",
+ getVariablesVariableIdChartsJson
+)
+// Deploy helpers
apiRouter.get("/deploys.json", async () => ({
deploys: await new DeployQueueServer().getDeploys(),
}))
@@ -2899,974 +432,4 @@ apiRouter.put("/deploy", async (req, res) => {
return triggerStaticBuild(res.locals.user, "Manually triggered deploy")
})
-getRouteWithROTransaction(apiRouter, "/gdocs", (req, res, trx) => {
- return getAllGdocIndexItemsOrderedByUpdatedAt(trx)
-})
-
-getRouteNonIdempotentWithRWTransaction(
- apiRouter,
- "/gdocs/:id",
- async (req, res, trx) => {
- const id = req.params.id
- const contentSource = req.query.contentSource as
- | GdocsContentSource
- | undefined
-
- try {
- // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published
- const gdoc = await getAndLoadGdocById(trx, id, contentSource)
-
- if (!gdoc.published) {
- await updateGdocContentOnly(trx, id, gdoc)
- }
-
- res.set("Cache-Control", "no-store")
- res.send(gdoc)
- } catch (error) {
- console.error("Error fetching gdoc", error)
- res.status(500).json({
- error: { message: String(error), status: 500 },
- })
- }
- }
-)
-
-/**
- * Handles all four `GdocPublishingAction` cases
- * - SavingDraft (no action)
- * - Publishing (index and bake)
- * - Updating (index and bake (potentially via lightning deploy))
- * - Unpublishing (remove from index and bake)
- */
-async function indexAndBakeGdocIfNeccesary(
- trx: db.KnexReadWriteTransaction,
- user: Required,
- prevGdoc:
- | GdocPost
- | GdocDataInsight
- | GdocHomepage
- | GdocAbout
- | GdocAuthor,
- nextGdoc: GdocPost | GdocDataInsight | GdocHomepage | GdocAbout | GdocAuthor
-) {
- const prevJson = prevGdoc.toJSON()
- const nextJson = nextGdoc.toJSON()
- const hasChanges = checkHasChanges(prevGdoc, nextGdoc)
- const action = getPublishingAction(prevJson, nextJson)
- const isGdocPost = checkIsGdocPostExcludingFragments(nextJson)
-
- await match(action)
- .with(GdocPublishingAction.SavingDraft, lodash.noop)
- .with(GdocPublishingAction.Publishing, async () => {
- if (isGdocPost) {
- await indexIndividualGdocPost(
- nextJson,
- trx,
- // If the gdoc is being published for the first time, prevGdoc.slug will be undefined
- // In that case, we pass nextJson.slug to see if it has any page views (i.e. from WP)
- prevGdoc.slug || nextJson.slug
- )
- }
- await triggerStaticBuild(user, `${action} ${nextJson.slug}`)
- })
- .with(GdocPublishingAction.Updating, async () => {
- if (isGdocPost) {
- await indexIndividualGdocPost(nextJson, trx, prevGdoc.slug)
- }
- if (checkIsLightningUpdate(prevJson, nextJson, hasChanges)) {
- await enqueueLightningChange(
- user,
- `Lightning update ${nextJson.slug}`,
- nextJson.slug
- )
- } else {
- await triggerStaticBuild(user, `${action} ${nextJson.slug}`)
- }
- })
- .with(GdocPublishingAction.Unpublishing, async () => {
- if (isGdocPost) {
- await removeIndividualGdocPostFromIndex(nextJson)
- }
- await triggerStaticBuild(user, `${action} ${nextJson.slug}`)
- })
- .exhaustive()
-}
-
-/**
- * Only supports creating a new empty Gdoc or updating an existing one. Does not
- * support creating a new Gdoc from an existing one. Relevant updates will
- * trigger a deploy.
- */
-putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => {
- const { id } = req.params
-
- if (isEmpty(req.body)) {
- return createOrLoadGdocById(trx, id)
- }
-
- const prevGdoc = await getAndLoadGdocById(trx, id)
- if (!prevGdoc) throw new JsonError(`No Google Doc with id ${id} found`)
-
- const nextGdoc = gdocFromJSON(req.body)
- await nextGdoc.loadState(trx)
-
- await addImagesToContentGraph(trx, nextGdoc)
-
- await setLinksForGdoc(
- trx,
- nextGdoc.id,
- nextGdoc.links,
- nextGdoc.published
- ? GdocLinkUpdateMode.DeleteAndInsert
- : GdocLinkUpdateMode.DeleteOnly
- )
-
- await upsertGdoc(trx, nextGdoc)
-
- await indexAndBakeGdocIfNeccesary(trx, res.locals.user, prevGdoc, nextGdoc)
-
- return nextGdoc
-})
-
-async function validateTombstoneRelatedLinkUrl(
- trx: db.KnexReadonlyTransaction,
- relatedLink?: string
-) {
- if (!relatedLink || !relatedLink.startsWith(GDOCS_BASE_URL)) return
- const id = relatedLink.match(gdocUrlRegex)?.[1]
- if (!id) {
- throw new JsonError(`Invalid related link: ${relatedLink}`)
- }
- const [gdoc] = await getMinimalGdocPostsByIds(trx, [id])
- if (!gdoc) {
- throw new JsonError(`Google Doc with ID ${id} not found`)
- }
- if (!gdoc.published) {
- throw new JsonError(`Google Doc with ID ${id} is not published`)
- }
-}
-
-deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => {
- const { id } = req.params
-
- const gdoc = await getGdocBaseObjectById(trx, id, false)
- if (!gdoc) throw new JsonError(`No Google Doc with id ${id} found`)
-
- const gdocSlug = getCanonicalUrl("", gdoc)
- const { tombstone } = req.body
-
- if (tombstone) {
- await validateTombstoneRelatedLinkUrl(trx, tombstone.relatedLinkUrl)
- const slug = gdocSlug.replace("/", "")
- const { relatedLinkThumbnail } = tombstone
- if (relatedLinkThumbnail) {
- const thumbnailExists = await db.checkIsImageInDB(
- trx,
- relatedLinkThumbnail
- )
- if (!thumbnailExists) {
- throw new JsonError(
- `Image with filename "${relatedLinkThumbnail}" not found`
- )
- }
- }
- await trx
- .table("posts_gdocs_tombstones")
- .insert({ ...tombstone, gdocId: id, slug })
- await trx
- .table("redirects")
- .insert({ source: gdocSlug, target: `/deleted${gdocSlug}` })
- }
-
- await trx
- .table("posts")
- .where({ gdocSuccessorId: gdoc.id })
- .update({ gdocSuccessorId: null })
-
- await trx.table(PostsGdocsLinksTableName).where({ sourceId: id }).delete()
- await trx.table(PostsGdocsXImagesTableName).where({ gdocId: id }).delete()
- await trx.table(PostsGdocsTableName).where({ id }).delete()
- await trx
- .table(PostsGdocsComponentsTableName)
- .where({ gdocId: id })
- .delete()
- if (gdoc.published && checkIsGdocPostExcludingFragments(gdoc)) {
- await removeIndividualGdocPostFromIndex(gdoc)
- }
- if (gdoc.published) {
- if (!tombstone && gdocSlug && gdocSlug !== "/") {
- // Assets have TTL of one week in Cloudflare. Add a redirect to make sure
- // the page is no longer accessible.
- // https://developers.cloudflare.com/pages/configuration/serving-pages/#asset-retention
- console.log(`Creating redirect for "${gdocSlug}" to "/"`)
- await db.knexRawInsert(
- trx,
- `INSERT INTO redirects (source, target, ttl)
- VALUES (?, ?, DATE_ADD(NOW(), INTERVAL 8 DAY))`,
- [gdocSlug, "/"]
- )
- }
- await triggerStaticBuild(res.locals.user, `Deleting ${gdocSlug}`)
- }
- return {}
-})
-
-postRouteWithRWTransaction(
- apiRouter,
- "/gdocs/:gdocId/setTags",
- async (req, res, trx) => {
- const { gdocId } = req.params
- const { tagIds } = req.body
- const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({
- id: id,
- }))
-
- await setTagsForGdoc(trx, gdocId, tagIdsAsObjects)
-
- return { success: true }
- }
-)
-
-getRouteNonIdempotentWithRWTransaction(
- apiRouter,
- "/images.json",
- async (_, res, trx) => {
- try {
- const images = await db.getCloudflareImages(trx)
- res.set("Cache-Control", "no-store")
- res.send({ images })
- } catch (error) {
- console.error("Error fetching images", error)
- res.status(500).json({
- error: { message: String(error), status: 500 },
- })
- }
- }
-)
-
-postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => {
- const { filename, type, content } = validateImagePayload(req.body)
-
- const { asBlob, dimensions, hash } = await processImageContent(
- content,
- type
- )
-
- const collision = await trx("images")
- .where({
- hash,
- replacedBy: null,
- })
- .first()
-
- if (collision) {
- return {
- success: false,
- error: `An image with this content already exists (filename: ${collision.filename})`,
- }
- }
-
- const preexisting = await trx("images")
- .where("filename", "=", filename)
- .first()
-
- if (preexisting) {
- return {
- success: false,
- error: "An image with this filename already exists",
- }
- }
-
- const cloudflareId = await uploadToCloudflare(filename, asBlob)
-
- if (!cloudflareId) {
- return {
- success: false,
- error: "Failed to upload image",
- }
- }
-
- await trx("images").insert({
- filename,
- originalWidth: dimensions.width,
- originalHeight: dimensions.height,
- cloudflareId,
- updatedAt: new Date().getTime(),
- userId: res.locals.user.id,
- hash,
- })
-
- const image = await db.getCloudflareImage(trx, filename)
-
- return {
- success: true,
- image,
- }
-})
-
-/**
- * Similar to the POST route, but for updating an existing image.
- * Creates a new image entry in the database and uploads the new image to Cloudflare.
- * The old image is marked as replaced by the new image.
- */
-putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => {
- const { type, content } = validateImagePayload(req.body)
- const { asBlob, dimensions, hash } = await processImageContent(
- content,
- type
- )
- const collision = await trx("images")
- .where({
- hash,
- replacedBy: null,
- })
- .first()
-
- if (collision) {
- return {
- success: false,
- error: `An exact copy of this image already exists (filename: ${collision.filename})`,
- }
- }
-
- const { id } = req.params
-
- const image = await trx("images")
- .where("id", "=", id)
- .first()
-
- if (!image) {
- throw new JsonError(`No image found for id ${id}`, 404)
- }
-
- const originalCloudflareId = image.cloudflareId
- const originalFilename = image.filename
- const originalAltText = image.defaultAlt
-
- if (!originalCloudflareId) {
- throw new JsonError(
- `Image with id ${id} has no associated Cloudflare image`,
- 400
- )
- }
-
- const newCloudflareId = await uploadToCloudflare(originalFilename, asBlob)
-
- if (!newCloudflareId) {
- throw new JsonError("Failed to upload image", 500)
- }
-
- const [newImageId] = await trx("images").insert({
- filename: originalFilename,
- originalWidth: dimensions.width,
- originalHeight: dimensions.height,
- cloudflareId: newCloudflareId,
- updatedAt: new Date().getTime(),
- userId: res.locals.user.id,
- defaultAlt: originalAltText,
- hash,
- version: image.version + 1,
- })
-
- await trx("images").where("id", "=", id).update({
- replacedBy: newImageId,
- })
-
- const updated = await db.getCloudflareImage(trx, originalFilename)
-
- await triggerStaticBuild(
- res.locals.user,
- `Updating image "${originalFilename}"`
- )
-
- return {
- success: true,
- image: updated,
- }
-})
-
-// Update alt text via patch
-patchRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => {
- const { id } = req.params
-
- const image = await trx("images")
- .where("id", "=", id)
- .first()
-
- if (!image) {
- throw new JsonError(`No image found for id ${id}`, 404)
- }
-
- const patchableImageProperties = ["defaultAlt"] as const
- const patch = lodash.pick(req.body, patchableImageProperties)
-
- if (Object.keys(patch).length === 0) {
- throw new JsonError("No patchable properties provided", 400)
- }
-
- await trx("images").where({ id }).update(patch)
-
- const updated = await trx("images")
- .where("id", "=", id)
- .first()
-
- return {
- success: true,
- image: updated,
- }
-})
-
-deleteRouteWithRWTransaction(apiRouter, "/images/:id", async (req, _, trx) => {
- const { id } = req.params
-
- const image = await trx("images")
- .where("id", "=", id)
- .first()
-
- if (!image) {
- throw new JsonError(`No image found for id ${id}`, 404)
- }
- if (!image.cloudflareId) {
- throw new JsonError(`Image does not have a cloudflare ID`, 400)
- }
-
- const replacementChain = await db.selectReplacementChainForImage(trx, id)
-
- await pMap(
- replacementChain,
- async (image) => {
- if (image.cloudflareId) {
- await deleteFromCloudflare(image.cloudflareId)
- }
- },
- { concurrency: 5 }
- )
-
- // There's an ON DELETE CASCADE which will delete the replacements
- await trx("images").where({ id }).delete()
-
- return {
- success: true,
- }
-})
-
-getRouteWithROTransaction(apiRouter, "/images/usage", async (_, __, trx) => {
- const usage = await db.getImageUsage(trx)
-
- return {
- success: true,
- usage,
- }
-})
-
-getRouteWithROTransaction(
- apiRouter,
- `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`,
- async (
- req: Request,
- res,
- trx
- ): Promise> => {
- const chartId = parseIntOrUndefined(req.params.chartId)
- if (!chartId) throw new JsonError(`Invalid chart ID`, 400)
-
- const topics = await getGptTopicSuggestions(trx, chartId)
-
- if (!topics.length)
- throw new JsonError(
- `No GPT topic suggestions found for chart ${chartId}`,
- 404
- )
-
- return {
- topics,
- }
- }
-)
-
-getRouteWithROTransaction(
- apiRouter,
- `/gpt/suggest-alt-text/:imageId`,
- async (
- req: Request,
- res,
- trx
- ): Promise<{
- success: boolean
- altText: string | null
- }> => {
- const imageId = parseIntOrUndefined(req.params.imageId)
- if (!imageId) throw new JsonError(`Invalid image ID`, 400)
- const image = await trx("images")
- .where("id", imageId)
- .first()
- if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404)
-
- const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public`
- let altText: string | null = ""
- try {
- altText = await fetchGptGeneratedAltText(src)
- } catch (error) {
- console.error(
- `Error fetching GPT alt text for image ${imageId}`,
- error
- )
- throw new JsonError(`Error fetching GPT alt text: ${error}`, 500)
- }
-
- if (!altText) {
- throw new JsonError(`Unable to generate alt text for image`, 404)
- }
-
- return { success: true, altText }
- }
-)
-
-postRouteWithRWTransaction(
- apiRouter,
- "/explorer/:slug/tags",
- async (req, res, trx) => {
- const { slug } = req.params
- const { tagIds } = req.body
- const explorer = await trx.table("explorers").where({ slug }).first()
- if (!explorer)
- throw new JsonError(`No explorer found for slug ${slug}`, 404)
-
- await trx.table("explorer_tags").where({ explorerSlug: slug }).delete()
- for (const tagId of tagIds) {
- await trx
- .table("explorer_tags")
- .insert({ explorerSlug: slug, tagId })
- }
-
- return { success: true }
- }
-)
-
-deleteRouteWithRWTransaction(
- apiRouter,
- "/explorer/:slug/tags",
- async (req: Request, res, trx) => {
- const { slug } = req.params
- await trx.table("explorer_tags").where({ explorerSlug: slug }).delete()
- return { success: true }
- }
-)
-
-// Get an ArchieML output of all the work produced by an author. This includes
-// gdoc articles, gdoc modular/linear topic pages and wordpress modular topic
-// pages. Data insights are excluded. This is used to manually populate the
-// [.secondary] section of the {.research-and-writing} block of author pages
-// using the alternate template, which highlights topics rather than articles.
-getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => {
- type WordpressPageRecord = {
- isWordpressPage: number
- } & Record<
- "slug" | "title" | "subtitle" | "thumbnail" | "authors" | "publishedAt",
- string
- >
- type GdocRecord = Pick
-
- const author = req.query.author || "Max Roser"
- const gdocs = await db.knexRaw(
- trx,
- `-- sql
- SELECT id, publishedAt
- FROM posts_gdocs
- WHERE JSON_CONTAINS(content->'$.authors', '"${author}"')
- AND type NOT IN ("data-insight", "fragment")
- AND published = 1
- `
- )
-
- // type: page
- const wpModularTopicPages = await db.knexRaw(
- trx,
- `-- sql
- SELECT
- wpApiSnapshot->>"$.slug" as slug,
- wpApiSnapshot->>"$.title.rendered" as title,
- wpApiSnapshot->>"$.excerpt.rendered" as subtitle,
- TRUE as isWordpressPage,
- wpApiSnapshot->>"$.authors_name" as authors,
- wpApiSnapshot->>"$.featured_media_paths.medium_large" as thumbnail,
- wpApiSnapshot->>"$.date" as publishedAt
- FROM posts p
- WHERE wpApiSnapshot->>"$.content" LIKE '%topic-page%'
- AND JSON_CONTAINS(wpApiSnapshot->'$.authors_name', '"${author}"')
- AND wpApiSnapshot->>"$.status" = 'publish'
- AND NOT EXISTS (
- SELECT 1 FROM posts_gdocs pg
- WHERE pg.slug = p.slug
- AND pg.content->>'$.type' LIKE '%topic-page'
- )
- `
- )
-
- const isWordpressPage = (
- post: WordpressPageRecord | GdocRecord
- ): post is WordpressPageRecord =>
- (post as WordpressPageRecord).isWordpressPage === 1
-
- function* generateProperty(key: string, value: string) {
- yield `${key}: ${value}\n`
- }
-
- const sortByDateDesc = (
- a: GdocRecord | WordpressPageRecord,
- b: GdocRecord | WordpressPageRecord
- ): number => {
- if (!a.publishedAt || !b.publishedAt) return 0
- return (
- new Date(b.publishedAt).getTime() -
- new Date(a.publishedAt).getTime()
- )
- }
-
- function* generateAllWorkArchieMl() {
- for (const post of [...gdocs, ...wpModularTopicPages].sort(
- sortByDateDesc
- )) {
- if (isWordpressPage(post)) {
- yield* generateProperty(
- "url",
- `https://ourworldindata.org/${post.slug}`
- )
- yield* generateProperty("title", post.title)
- yield* generateProperty("subtitle", post.subtitle)
- yield* generateProperty(
- "authors",
- JSON.parse(post.authors).join(", ")
- )
- const parsedPath = path.parse(post.thumbnail)
- yield* generateProperty(
- "filename",
- // /app/uploads/2021/09/reducing-fertilizer-768x301.png -> reducing-fertilizer.png
- path.format({
- name: parsedPath.name.replace(/-\d+x\d+$/, ""),
- ext: parsedPath.ext,
- })
- )
- yield "\n"
- } else {
- // this is a gdoc
- yield* generateProperty(
- "url",
- `https://docs.google.com/document/d/${post.id}/edit`
- )
- yield "\n"
- }
- }
- }
-
- res.type("text/plain")
- return [...generateAllWorkArchieMl()].join("")
-})
-
-getRouteWithROTransaction(
- apiRouter,
- "/flatTagGraph.json",
- async (req, res, trx) => {
- const flatTagGraph = await db.getFlatTagGraph(trx)
- return flatTagGraph
- }
-)
-
-postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => {
- const tagGraph = req.body?.tagGraph as unknown
- if (!tagGraph) {
- throw new JsonError("No tagGraph provided", 400)
- }
-
- function validateFlatTagGraph(
- tagGraph: Record
- ): tagGraph is FlatTagGraph {
- if (lodash.isObject(tagGraph)) {
- for (const [key, value] of Object.entries(tagGraph)) {
- if (!lodash.isString(key) && isNaN(Number(key))) {
- return false
- }
- if (!lodash.isArray(value)) {
- return false
- }
- for (const tag of value) {
- if (
- !(
- checkIsPlainObjectWithGuard(tag) &&
- lodash.isNumber(tag.weight) &&
- lodash.isNumber(tag.parentId) &&
- lodash.isNumber(tag.childId)
- )
- ) {
- return false
- }
- }
- }
- }
-
- return true
- }
- const isValid = validateFlatTagGraph(tagGraph)
- if (!isValid) {
- throw new JsonError("Invalid tag graph provided", 400)
- }
- await db.updateTagGraph(trx, tagGraph)
- res.send({ success: true })
-})
-
-const createPatchConfigAndQueryParamsForChartView = async (
- knex: db.KnexReadonlyTransaction,
- parentChartId: number,
- config: GrapherInterface
-) => {
- const parentChartConfig = await expectChartById(knex, parentChartId)
-
- config = omit(config, CHART_VIEW_PROPS_TO_OMIT)
-
- const patchToParentChart = diffGrapherConfigs(config, parentChartConfig)
-
- const fullConfigIncludingDefaults = mergeGrapherConfigs(
- defaultGrapherConfig,
- config
- )
- const patchConfigToSave = {
- ...patchToParentChart,
-
- // We want to make sure we're explicitly persisting some props like entity selection
- // always, so they never change when the parent chart changes.
- // For this, we need to ensure we include the default layer, so that we even
- // persist these props when they are the same as the default.
- ...pick(fullConfigIncludingDefaults, CHART_VIEW_PROPS_TO_PERSIST),
- }
-
- const queryParams = grapherConfigToQueryParams(config)
-
- const fullConfig = mergeGrapherConfigs(parentChartConfig, patchConfigToSave)
- return { patchConfig: patchConfigToSave, fullConfig, queryParams }
-}
-
-getRouteWithROTransaction(apiRouter, "/chartViews", async (req, res, trx) => {
- type ChartViewRow = Pick & {
- lastEditedByUser: string
- chartConfigId: string
- title: string
- parentChartId: number
- parentTitle: string
- }
-
- const rows: ChartViewRow[] = await db.knexRaw(
- trx,
- `-- sql
- SELECT
- cv.id,
- cv.name,
- cv.updatedAt,
- u.fullName as lastEditedByUser,
- cv.chartConfigId,
- cc.full ->> "$.title" as title,
- cv.parentChartId,
- pcc.full ->> "$.title" as parentTitle
- FROM chart_views cv
- JOIN chart_configs cc ON cv.chartConfigId = cc.id
- JOIN charts pc ON cv.parentChartId = pc.id
- JOIN chart_configs pcc ON pc.configId = pcc.id
- JOIN users u ON cv.lastEditedByUserId = u.id
- ORDER BY cv.updatedAt DESC
- `
- )
-
- const chartViews: ApiChartViewOverview[] = rows.map((row) => ({
- id: row.id,
- name: row.name,
- updatedAt: row.updatedAt?.toISOString() ?? null,
- lastEditedByUser: row.lastEditedByUser,
- chartConfigId: row.chartConfigId,
- title: row.title,
- parent: {
- id: row.parentChartId,
- title: row.parentTitle,
- },
- }))
-
- return { chartViews }
-})
-
-getRouteWithROTransaction(
- apiRouter,
- "/chartViews/:id",
- async (req, res, trx) => {
- const id = expectInt(req.params.id)
-
- type ChartViewRow = Pick<
- DbPlainChartView,
- "id" | "name" | "updatedAt"
- > & {
- lastEditedByUser: string
- chartConfigId: string
- configFull: JsonString
- configPatch: JsonString
- parentChartId: number
- parentConfigFull: JsonString
- queryParamsForParentChart: JsonString
- }
-
- const row = await db.knexRawFirst(
- trx,
- `-- sql
- SELECT
- cv.id,
- cv.name,
- cv.updatedAt,
- u.fullName as lastEditedByUser,
- cv.chartConfigId,
- cc.full as configFull,
- cc.patch as configPatch,
- cv.parentChartId,
- pcc.full as parentConfigFull,
- cv.queryParamsForParentChart
- FROM chart_views cv
- JOIN chart_configs cc ON cv.chartConfigId = cc.id
- JOIN charts pc ON cv.parentChartId = pc.id
- JOIN chart_configs pcc ON pc.configId = pcc.id
- JOIN users u ON cv.lastEditedByUserId = u.id
- WHERE cv.id = ?
- `,
- [id]
- )
-
- if (!row) {
- throw new JsonError(`No chart view found for id ${id}`, 404)
- }
-
- const chartView = {
- ...row,
- configFull: parseChartConfig(row.configFull),
- configPatch: parseChartConfig(row.configPatch),
- parentConfigFull: parseChartConfig(row.parentConfigFull),
- queryParamsForParentChart: JSON.parse(
- row.queryParamsForParentChart
- ),
- }
-
- return chartView
- }
-)
-
-postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => {
- const { name, parentChartId } = req.body as Pick<
- DbPlainChartView,
- "name" | "parentChartId"
- >
- const rawConfig = req.body.config as GrapherInterface
- if (!name || !parentChartId || !rawConfig) {
- throw new JsonError("Invalid request", 400)
- }
-
- const { patchConfig, fullConfig, queryParams } =
- await createPatchConfigAndQueryParamsForChartView(
- trx,
- parentChartId,
- rawConfig
- )
-
- const { chartConfigId } = await saveNewChartConfigInDbAndR2(
- trx,
- undefined,
- patchConfig,
- fullConfig
- )
-
- // insert into chart_views
- const insertRow: DbInsertChartView = {
- name,
- parentChartId,
- lastEditedByUserId: res.locals.user.id,
- chartConfigId: chartConfigId,
- queryParamsForParentChart: JSON.stringify(queryParams),
- }
- const result = await trx.table(ChartViewsTableName).insert(insertRow)
- const [resultId] = result
-
- return { chartViewId: resultId, success: true }
-})
-
-putRouteWithRWTransaction(
- apiRouter,
- "/chartViews/:id",
- async (req, res, trx) => {
- const id = expectInt(req.params.id)
- const rawConfig = req.body.config as GrapherInterface
- if (!rawConfig) {
- throw new JsonError("Invalid request", 400)
- }
-
- const existingRow: Pick<
- DbPlainChartView,
- "chartConfigId" | "parentChartId"
- > = await trx(ChartViewsTableName)
- .select("parentChartId", "chartConfigId")
- .where({ id })
- .first()
-
- if (!existingRow) {
- throw new JsonError(`No chart view found for id ${id}`, 404)
- }
-
- const { patchConfig, fullConfig, queryParams } =
- await createPatchConfigAndQueryParamsForChartView(
- trx,
- existingRow.parentChartId,
- rawConfig
- )
-
- await updateChartConfigInDbAndR2(
- trx,
- existingRow.chartConfigId as Base64String,
- patchConfig,
- fullConfig
- )
-
- // update chart_views
- await trx
- .table(ChartViewsTableName)
- .where({ id })
- .update({
- updatedAt: new Date(),
- lastEditedByUserId: res.locals.user.id,
- queryParamsForParentChart: JSON.stringify(queryParams),
- })
-
- return { success: true }
- }
-)
-
-deleteRouteWithRWTransaction(
- apiRouter,
- "/chartViews/:id",
- async (req, res, trx) => {
- const id = expectInt(req.params.id)
-
- const chartConfigId: string | undefined = await trx(ChartViewsTableName)
- .select("chartConfigId")
- .where({ id })
- .first()
- .then((row) => row?.chartConfigId)
-
- if (!chartConfigId) {
- throw new JsonError(`No chart view found for id ${id}`, 404)
- }
-
- await trx.table(ChartViewsTableName).where({ id }).delete()
-
- await deleteGrapherConfigFromR2ByUUID(chartConfigId)
-
- await trx
- .table(ChartConfigsTableName)
- .where({ id: chartConfigId })
- .delete()
-
- return { success: true }
- }
-)
-
export { apiRouter }
diff --git a/adminSiteServer/apiRoutes/bulkUpdates.ts b/adminSiteServer/apiRoutes/bulkUpdates.ts
new file mode 100644
index 00000000000..82aa5b598c4
--- /dev/null
+++ b/adminSiteServer/apiRoutes/bulkUpdates.ts
@@ -0,0 +1,242 @@
+import {
+ DbPlainChart,
+ DbRawChartConfig,
+ GrapherInterface,
+ DbRawVariable,
+} from "@ourworldindata/types"
+import { parseIntOrUndefined } from "@ourworldindata/utils"
+import {
+ BulkGrapherConfigResponse,
+ BulkChartEditResponseRow,
+ chartBulkUpdateAllowedColumnNamesAndTypes,
+ GrapherConfigPatch,
+ VariableAnnotationsResponseRow,
+ variableAnnotationAllowedColumnNamesAndTypes,
+} from "../../adminShared/AdminSessionTypes.js"
+import { applyPatch } from "../../adminShared/patchHelper.js"
+import {
+ OperationContext,
+ parseToOperation,
+} from "../../adminShared/SqlFilterSExpression.js"
+import {
+ getGrapherConfigsForVariable,
+ updateGrapherConfigAdminOfVariable,
+} from "../../db/model/Variable.js"
+import { saveGrapher } from "./charts.js"
+import * as db from "../../db/db.js"
+import * as lodash from "lodash"
+import { Request } from "../authentication.js"
+import e from "express"
+
+export async function getChartBulkUpdate(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+): Promise> {
+ const context: OperationContext = {
+ grapherConfigFieldName: "chart_configs.full",
+ whitelistedColumnNamesAndTypes:
+ chartBulkUpdateAllowedColumnNamesAndTypes,
+ }
+ const filterSExpr =
+ req.query.filter !== undefined
+ ? parseToOperation(req.query.filter as string, context)
+ : undefined
+
+ const offset = parseIntOrUndefined(req.query.offset as string) ?? 0
+
+ // Note that our DSL generates sql here that we splice directly into the SQL as text
+ // This is a potential for a SQL injection attack but we control the DSL and are
+ // careful there to only allow carefully guarded vocabularies from being used, not
+ // arbitrary user input
+ const whereClause = filterSExpr?.toSql() ?? "true"
+ const resultsWithStringGrapherConfigs = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT
+ charts.id as id,
+ chart_configs.full as config,
+ charts.createdAt as createdAt,
+ charts.updatedAt as updatedAt,
+ charts.lastEditedAt as lastEditedAt,
+ charts.publishedAt as publishedAt,
+ lastEditedByUser.fullName as lastEditedByUser,
+ publishedByUser.fullName as publishedByUser
+ FROM charts
+ LEFT JOIN chart_configs ON chart_configs.id = charts.configId
+ LEFT JOIN users lastEditedByUser ON lastEditedByUser.id=charts.lastEditedByUserId
+ LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId
+ WHERE ${whereClause}
+ ORDER BY charts.id DESC
+ LIMIT 50
+ OFFSET ${offset.toString()}
+ `
+ )
+
+ const results = resultsWithStringGrapherConfigs.map((row: any) => ({
+ ...row,
+ config: lodash.isNil(row.config) ? null : JSON.parse(row.config),
+ }))
+ const resultCount = await db.knexRaw<{ count: number }>(
+ trx,
+ `-- sql
+ SELECT count(*) as count
+ FROM charts
+ JOIN chart_configs ON chart_configs.id = charts.configId
+ WHERE ${whereClause}
+ `
+ )
+ return { rows: results, numTotalRows: resultCount[0].count }
+}
+
+export async function updateBulkChartConfigs(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const patchesList = req.body as GrapherConfigPatch[]
+ const chartIds = new Set(patchesList.map((patch) => patch.id))
+
+ const configsAndIds = await db.knexRaw<
+ Pick & { config: DbRawChartConfig["full"] }
+ >(
+ trx,
+ `-- sql
+ SELECT c.id, cc.full as config
+ FROM charts c
+ JOIN chart_configs cc ON cc.id = c.configId
+ WHERE c.id IN (?)
+ `,
+ [[...chartIds.values()]]
+ )
+ const configMap = new Map(
+ configsAndIds.map((item: any) => [
+ item.id,
+ // make sure that the id is set, otherwise the update behaviour is weird
+ // TODO: discuss if this has unintended side effects
+ item.config ? { ...JSON.parse(item.config), id: item.id } : {},
+ ])
+ )
+ const oldValuesConfigMap = new Map(configMap)
+ // console.log("ids", configsAndIds.map((item : any) => item.id))
+ for (const patchSet of patchesList) {
+ const config = configMap.get(patchSet.id)
+ configMap.set(patchSet.id, applyPatch(patchSet, config))
+ }
+
+ for (const [id, newConfig] of configMap.entries()) {
+ await saveGrapher(trx, {
+ user: res.locals.user,
+ newConfig,
+ existingConfig: oldValuesConfigMap.get(id),
+ referencedVariablesMightChange: false,
+ })
+ }
+
+ return { success: true }
+}
+
+export async function getVariableAnnotations(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+): Promise> {
+ const context: OperationContext = {
+ grapherConfigFieldName: "grapherConfigAdmin",
+ whitelistedColumnNamesAndTypes:
+ variableAnnotationAllowedColumnNamesAndTypes,
+ }
+ const filterSExpr =
+ req.query.filter !== undefined
+ ? parseToOperation(req.query.filter as string, context)
+ : undefined
+
+ const offset = parseIntOrUndefined(req.query.offset as string) ?? 0
+
+ // Note that our DSL generates sql here that we splice directly into the SQL as text
+ // This is a potential for a SQL injection attack but we control the DSL and are
+ // careful there to only allow carefully guarded vocabularies from being used, not
+ // arbitrary user input
+ const whereClause = filterSExpr?.toSql() ?? "true"
+ const resultsWithStringGrapherConfigs = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT
+ variables.id as id,
+ variables.name as name,
+ chart_configs.patch as config,
+ d.name as datasetname,
+ namespaces.name as namespacename,
+ variables.createdAt as createdAt,
+ variables.updatedAt as updatedAt,
+ variables.description as description
+ FROM variables
+ LEFT JOIN active_datasets as d on variables.datasetId = d.id
+ LEFT JOIN namespaces on d.namespace = namespaces.name
+ LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id
+ WHERE ${whereClause}
+ ORDER BY variables.id DESC
+ LIMIT 50
+ OFFSET ${offset.toString()}
+ `
+ )
+
+ const results = resultsWithStringGrapherConfigs.map((row: any) => ({
+ ...row,
+ config: lodash.isNil(row.config) ? null : JSON.parse(row.config),
+ }))
+ const resultCount = await db.knexRaw<{ count: number }>(
+ trx,
+ `-- sql
+ SELECT count(*) as count
+ FROM variables
+ LEFT JOIN active_datasets as d on variables.datasetId = d.id
+ LEFT JOIN namespaces on d.namespace = namespaces.name
+ LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id
+ WHERE ${whereClause}
+ `
+ )
+ return { rows: results, numTotalRows: resultCount[0].count }
+}
+
+export async function updateVariableAnnotations(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const patchesList = req.body as GrapherConfigPatch[]
+ const variableIds = new Set(patchesList.map((patch) => patch.id))
+
+ const configsAndIds = await db.knexRaw<
+ Pick & {
+ grapherConfigAdmin: DbRawChartConfig["patch"]
+ }
+ >(
+ trx,
+ `-- sql
+ SELECT v.id, cc.patch AS grapherConfigAdmin
+ FROM variables v
+ LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id
+ WHERE v.id IN (?)`,
+ [[...variableIds.values()]]
+ )
+ const configMap = new Map(
+ configsAndIds.map((item: any) => [
+ item.id,
+ item.grapherConfigAdmin ? JSON.parse(item.grapherConfigAdmin) : {},
+ ])
+ )
+ // console.log("ids", configsAndIds.map((item : any) => item.id))
+ for (const patchSet of patchesList) {
+ const config = configMap.get(patchSet.id)
+ configMap.set(patchSet.id, applyPatch(patchSet, config))
+ }
+
+ for (const [variableId, newConfig] of configMap.entries()) {
+ const variable = await getGrapherConfigsForVariable(trx, variableId)
+ if (!variable) continue
+ await updateGrapherConfigAdminOfVariable(trx, variable, newConfig)
+ }
+
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/chartViews.ts b/adminSiteServer/apiRoutes/chartViews.ts
new file mode 100644
index 00000000000..bccad837766
--- /dev/null
+++ b/adminSiteServer/apiRoutes/chartViews.ts
@@ -0,0 +1,293 @@
+import {
+ defaultGrapherConfig,
+ grapherConfigToQueryParams,
+} from "@ourworldindata/grapher"
+import {
+ GrapherInterface,
+ CHART_VIEW_PROPS_TO_OMIT,
+ CHART_VIEW_PROPS_TO_PERSIST,
+ DbPlainChartView,
+ JsonString,
+ JsonError,
+ parseChartConfig,
+ DbInsertChartView,
+ ChartViewsTableName,
+ Base64String,
+ ChartConfigsTableName,
+} from "@ourworldindata/types"
+import { diffGrapherConfigs, mergeGrapherConfigs } from "@ourworldindata/utils"
+import { omit, pick } from "lodash"
+import { ApiChartViewOverview } from "../../adminShared/AdminTypes.js"
+import { expectInt } from "../../serverUtils/serverUtil.js"
+import {
+ saveNewChartConfigInDbAndR2,
+ updateChartConfigInDbAndR2,
+} from "../chartConfigHelpers.js"
+import { deleteGrapherConfigFromR2ByUUID } from "../chartConfigR2Helpers.js"
+
+import * as db from "../../db/db.js"
+import { expectChartById } from "./charts.js"
+import { Request } from "../authentication.js"
+import e from "express"
+const createPatchConfigAndQueryParamsForChartView = async (
+ knex: db.KnexReadonlyTransaction,
+ parentChartId: number,
+ config: GrapherInterface
+) => {
+ const parentChartConfig = await expectChartById(knex, parentChartId)
+
+ config = omit(config, CHART_VIEW_PROPS_TO_OMIT)
+
+ const patchToParentChart = diffGrapherConfigs(config, parentChartConfig)
+
+ const fullConfigIncludingDefaults = mergeGrapherConfigs(
+ defaultGrapherConfig,
+ config
+ )
+ const patchConfigToSave = {
+ ...patchToParentChart,
+
+ // We want to make sure we're explicitly persisting some props like entity selection
+ // always, so they never change when the parent chart changes.
+ // For this, we need to ensure we include the default layer, so that we even
+ // persist these props when they are the same as the default.
+ ...pick(fullConfigIncludingDefaults, CHART_VIEW_PROPS_TO_PERSIST),
+ }
+
+ const queryParams = grapherConfigToQueryParams(patchConfigToSave)
+
+ const fullConfig = mergeGrapherConfigs(parentChartConfig, patchConfigToSave)
+ return { patchConfig: patchConfigToSave, fullConfig, queryParams }
+}
+
+export async function getChartViews(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ type ChartViewRow = Pick & {
+ lastEditedByUser: string
+ chartConfigId: string
+ title: string
+ parentChartId: number
+ parentTitle: string
+ }
+
+ const rows: ChartViewRow[] = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT
+ cv.id,
+ cv.name,
+ cv.updatedAt,
+ u.fullName as lastEditedByUser,
+ cv.chartConfigId,
+ cc.full ->> "$.title" as title,
+ cv.parentChartId,
+ pcc.full ->> "$.title" as parentTitle
+ FROM chart_views cv
+ JOIN chart_configs cc ON cv.chartConfigId = cc.id
+ JOIN charts pc ON cv.parentChartId = pc.id
+ JOIN chart_configs pcc ON pc.configId = pcc.id
+ JOIN users u ON cv.lastEditedByUserId = u.id
+ ORDER BY cv.updatedAt DESC
+ `
+ )
+
+ const chartViews: ApiChartViewOverview[] = rows.map((row) => ({
+ id: row.id,
+ name: row.name,
+ updatedAt: row.updatedAt?.toISOString() ?? null,
+ lastEditedByUser: row.lastEditedByUser,
+ chartConfigId: row.chartConfigId,
+ title: row.title,
+ parent: {
+ id: row.parentChartId,
+ title: row.parentTitle,
+ },
+ }))
+
+ return { chartViews }
+}
+
+export async function getChartViewById(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const id = expectInt(req.params.id)
+
+ type ChartViewRow = Pick & {
+ lastEditedByUser: string
+ chartConfigId: string
+ configFull: JsonString
+ configPatch: JsonString
+ parentChartId: number
+ parentConfigFull: JsonString
+ queryParamsForParentChart: JsonString
+ }
+
+ const row = await db.knexRawFirst(
+ trx,
+ `-- sql
+ SELECT
+ cv.id,
+ cv.name,
+ cv.updatedAt,
+ u.fullName as lastEditedByUser,
+ cv.chartConfigId,
+ cc.full as configFull,
+ cc.patch as configPatch,
+ cv.parentChartId,
+ pcc.full as parentConfigFull,
+ cv.queryParamsForParentChart
+ FROM chart_views cv
+ JOIN chart_configs cc ON cv.chartConfigId = cc.id
+ JOIN charts pc ON cv.parentChartId = pc.id
+ JOIN chart_configs pcc ON pc.configId = pcc.id
+ JOIN users u ON cv.lastEditedByUserId = u.id
+ WHERE cv.id = ?
+ `,
+ [id]
+ )
+
+ if (!row) {
+ throw new JsonError(`No chart view found for id ${id}`, 404)
+ }
+
+ const chartView = {
+ ...row,
+ configFull: parseChartConfig(row.configFull),
+ configPatch: parseChartConfig(row.configPatch),
+ parentConfigFull: parseChartConfig(row.parentConfigFull),
+ queryParamsForParentChart: JSON.parse(row.queryParamsForParentChart),
+ }
+
+ return chartView
+}
+
+export async function createChartView(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const { name, parentChartId } = req.body as Pick<
+ DbPlainChartView,
+ "name" | "parentChartId"
+ >
+ const rawConfig = req.body.config as GrapherInterface
+ if (!name || !parentChartId || !rawConfig) {
+ throw new JsonError("Invalid request", 400)
+ }
+ const chartViewWithName = await trx
+ .table(ChartViewsTableName)
+ .where({ name })
+ .first()
+ if (chartViewWithName) {
+ return {
+ success: false,
+ errorMsg: `Narrative chart with name "${name}" already exists`,
+ }
+ }
+
+ const { patchConfig, fullConfig, queryParams } =
+ await createPatchConfigAndQueryParamsForChartView(
+ trx,
+ parentChartId,
+ rawConfig
+ )
+
+ const { chartConfigId } = await saveNewChartConfigInDbAndR2(
+ trx,
+ undefined,
+ patchConfig,
+ fullConfig
+ )
+ // insert into chart_views
+ const insertRow: DbInsertChartView = {
+ name,
+ parentChartId,
+ lastEditedByUserId: res.locals.user.id,
+ chartConfigId: chartConfigId,
+ queryParamsForParentChart: JSON.stringify(queryParams),
+ }
+ const result = await trx.table(ChartViewsTableName).insert(insertRow)
+ const [resultId] = result
+
+ return { chartViewId: resultId, success: true }
+}
+
+export async function updateChartView(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const id = expectInt(req.params.id)
+ const rawConfig = req.body.config as GrapherInterface
+ if (!rawConfig) {
+ throw new JsonError("Invalid request", 400)
+ }
+
+ const existingRow: Pick<
+ DbPlainChartView,
+ "chartConfigId" | "parentChartId"
+ > = await trx(ChartViewsTableName)
+ .select("parentChartId", "chartConfigId")
+ .where({ id })
+ .first()
+
+ if (!existingRow) {
+ throw new JsonError(`No chart view found for id ${id}`, 404)
+ }
+
+ const { patchConfig, fullConfig, queryParams } =
+ await createPatchConfigAndQueryParamsForChartView(
+ trx,
+ existingRow.parentChartId,
+ rawConfig
+ )
+
+ await updateChartConfigInDbAndR2(
+ trx,
+ existingRow.chartConfigId as Base64String,
+ patchConfig,
+ fullConfig
+ )
+
+ await trx
+ .table(ChartViewsTableName)
+ .where({ id })
+ .update({
+ updatedAt: new Date(),
+ lastEditedByUserId: res.locals.user.id,
+ queryParamsForParentChart: JSON.stringify(queryParams),
+ })
+
+ return { success: true }
+}
+
+export async function deleteChartView(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const id = expectInt(req.params.id)
+
+ const chartConfigId: string | undefined = await trx(ChartViewsTableName)
+ .select("chartConfigId")
+ .where({ id })
+ .first()
+ .then((row) => row?.chartConfigId)
+
+ if (!chartConfigId) {
+ throw new JsonError(`No chart view found for id ${id}`, 404)
+ }
+
+ await trx.table(ChartViewsTableName).where({ id }).delete()
+
+ await deleteGrapherConfigFromR2ByUUID(chartConfigId)
+
+ await trx.table(ChartConfigsTableName).where({ id: chartConfigId }).delete()
+
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/charts.ts b/adminSiteServer/apiRoutes/charts.ts
new file mode 100644
index 00000000000..2b19a204a8a
--- /dev/null
+++ b/adminSiteServer/apiRoutes/charts.ts
@@ -0,0 +1,809 @@
+import { migrateGrapherConfigToLatestVersion } from "@ourworldindata/grapher"
+import {
+ GrapherInterface,
+ JsonError,
+ DbPlainUser,
+ Base64String,
+ serializeChartConfig,
+ DbPlainChart,
+ DbPlainChartSlugRedirect,
+ R2GrapherConfigDirectory,
+ DbInsertChartRevision,
+ DbRawChartConfig,
+ ChartConfigsTableName,
+} from "@ourworldindata/types"
+import {
+ diffGrapherConfigs,
+ mergeGrapherConfigs,
+ parseIntOrUndefined,
+ omitUndefinedValues,
+} from "@ourworldindata/utils"
+import Papa from "papaparse"
+import { uuidv7 } from "uuidv7"
+import { References } from "../../adminSiteClient/AbstractChartEditor.js"
+import { ChartViewMinimalInformation } from "../../adminSiteClient/ChartEditor.js"
+import { denormalizeLatestCountryData } from "../../baker/countryProfiles.js"
+import {
+ getChartConfigById,
+ getPatchConfigByChartId,
+ getParentByChartConfig,
+ isInheritanceEnabledForChart,
+ OldChartFieldList,
+ oldChartFieldList,
+ assignTagsForCharts,
+ getParentByChartId,
+ getRedirectsByChartId,
+ getChartSlugById,
+ setChartTags,
+} from "../../db/model/Chart.js"
+import {
+ getWordpressPostReferencesByChartId,
+ getGdocsPostReferencesByChartId,
+} from "../../db/model/Post.js"
+import { expectInt, isValidSlug } from "../../serverUtils/serverUtil.js"
+import {
+ BAKED_BASE_URL,
+ ADMIN_BASE_URL,
+} from "../../settings/clientSettings.js"
+import {
+ retrieveChartConfigFromDbAndSaveToR2,
+ updateChartConfigInDbAndR2,
+} from "../chartConfigHelpers.js"
+import {
+ deleteGrapherConfigFromR2,
+ deleteGrapherConfigFromR2ByUUID,
+ saveGrapherConfigToR2ByUUID,
+} from "../chartConfigR2Helpers.js"
+import { triggerStaticBuild } from "./routeUtils.js"
+import * as db from "../../db/db.js"
+import { getLogsByChartId } from "../getLogsByChartId.js"
+import { getPublishedLinksTo } from "../../db/model/Link.js"
+
+import { Request } from "../authentication.js"
+import e from "express"
+export const getReferencesByChartId = async (
+ chartId: number,
+ knex: db.KnexReadonlyTransaction
+): Promise => {
+ const postsWordpressPromise = getWordpressPostReferencesByChartId(
+ chartId,
+ knex
+ )
+ const postGdocsPromise = getGdocsPostReferencesByChartId(chartId, knex)
+ const explorerSlugsPromise = db.knexRaw<{ explorerSlug: string }>(
+ knex,
+ `SELECT DISTINCT
+ explorerSlug
+ FROM
+ explorer_charts
+ WHERE
+ chartId = ?`,
+ [chartId]
+ )
+ const chartViewsPromise = db.knexRaw(
+ knex,
+ `-- sql
+ SELECT cv.id, cv.name, cc.full ->> "$.title" AS title
+ FROM chart_views cv
+ JOIN chart_configs cc ON cc.id = cv.chartConfigId
+ WHERE cv.parentChartId = ?`,
+ [chartId]
+ )
+ const [postsWordpress, postsGdocs, explorerSlugs, chartViews] =
+ await Promise.all([
+ postsWordpressPromise,
+ postGdocsPromise,
+ explorerSlugsPromise,
+ chartViewsPromise,
+ ])
+
+ return {
+ postsGdocs,
+ postsWordpress,
+ explorers: explorerSlugs.map(
+ (row: { explorerSlug: string }) => row.explorerSlug
+ ),
+ chartViews,
+ }
+}
+
+export const expectChartById = async (
+ knex: db.KnexReadonlyTransaction,
+ chartId: any
+): Promise => {
+ const chart = await getChartConfigById(knex, expectInt(chartId))
+ if (chart) return chart.config
+
+ throw new JsonError(`No chart found for id ${chartId}`, 404)
+}
+
+const expectPatchConfigByChartId = async (
+ knex: db.KnexReadonlyTransaction,
+ chartId: any
+): Promise => {
+ const patchConfig = await getPatchConfigByChartId(knex, expectInt(chartId))
+ if (!patchConfig) {
+ throw new JsonError(`No chart found for id ${chartId}`, 404)
+ }
+ return patchConfig
+}
+
+const saveNewChart = async (
+ knex: db.KnexReadWriteTransaction,
+ {
+ config,
+ user,
+ // new charts inherit by default
+ shouldInherit = true,
+ }: { config: GrapherInterface; user: DbPlainUser; shouldInherit?: boolean }
+): Promise<{
+ chartConfigId: Base64String
+ patchConfig: GrapherInterface
+ fullConfig: GrapherInterface
+}> => {
+ // grab the parent of the chart if inheritance should be enabled
+ const parent = shouldInherit
+ ? await getParentByChartConfig(knex, config)
+ : undefined
+
+ // compute patch and full configs
+ const patchConfig = diffGrapherConfigs(config, parent?.config ?? {})
+ const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig)
+
+ // insert patch & full configs into the chart_configs table
+ // We can't quite use `saveNewChartConfigInDbAndR2` here, because
+ // we need to update the chart id in the config after inserting it.
+ const chartConfigId = uuidv7() as Base64String
+ await db.knexRaw(
+ knex,
+ `-- sql
+ INSERT INTO chart_configs (id, patch, full)
+ VALUES (?, ?, ?)
+ `,
+ [
+ chartConfigId,
+ serializeChartConfig(patchConfig),
+ serializeChartConfig(fullConfig),
+ ]
+ )
+
+ // add a new chart to the charts table
+ const result = await db.knexRawInsert(
+ knex,
+ `-- sql
+ INSERT INTO charts (configId, isInheritanceEnabled, lastEditedAt, lastEditedByUserId)
+ VALUES (?, ?, ?, ?)
+ `,
+ [chartConfigId, shouldInherit, new Date(), user.id]
+ )
+
+ // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true
+ const chartId = result.insertId
+ patchConfig.id = chartId
+ fullConfig.id = chartId
+ await db.knexRaw(
+ knex,
+ `-- sql
+ UPDATE chart_configs cc
+ JOIN charts c ON c.configId = cc.id
+ SET
+ cc.patch=JSON_SET(cc.patch, '$.id', ?),
+ cc.full=JSON_SET(cc.full, '$.id', ?)
+ WHERE c.id = ?
+ `,
+ [chartId, chartId, chartId]
+ )
+
+ await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId)
+
+ return { chartConfigId, patchConfig, fullConfig }
+}
+
+const updateExistingChart = async (
+ knex: db.KnexReadWriteTransaction,
+ params: {
+ config: GrapherInterface
+ user: DbPlainUser
+ chartId: number
+ // if undefined, keep inheritance as is.
+ // if true or false, enable or disable inheritance
+ shouldInherit?: boolean
+ }
+): Promise<{
+ chartConfigId: Base64String
+ patchConfig: GrapherInterface
+ fullConfig: GrapherInterface
+}> => {
+ const { config, user, chartId } = params
+
+ // make sure that the id of the incoming config matches the chart id
+ config.id = chartId
+
+ // if inheritance is enabled, grab the parent from its config
+ const shouldInherit =
+ params.shouldInherit ??
+ (await isInheritanceEnabledForChart(knex, chartId))
+ const parent = shouldInherit
+ ? await getParentByChartConfig(knex, config)
+ : undefined
+
+ // compute patch and full configs
+ const patchConfig = diffGrapherConfigs(config, parent?.config ?? {})
+ const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig)
+
+ const chartConfigIdRow = await db.knexRawFirst<
+ Pick
+ >(knex, `SELECT configId FROM charts WHERE id = ?`, [chartId])
+
+ if (!chartConfigIdRow)
+ throw new JsonError(`No chart config found for id ${chartId}`, 404)
+
+ const now = new Date()
+
+ const { chartConfigId } = await updateChartConfigInDbAndR2(
+ knex,
+ chartConfigIdRow.configId as Base64String,
+ patchConfig,
+ fullConfig
+ )
+
+ // update charts row
+ await db.knexRaw(
+ knex,
+ `-- sql
+ UPDATE charts
+ SET isInheritanceEnabled=?, updatedAt=?, lastEditedAt=?, lastEditedByUserId=?
+ WHERE id = ?
+ `,
+ [shouldInherit, now, now, user.id, chartId]
+ )
+
+ return { chartConfigId, patchConfig, fullConfig }
+}
+
+export const saveGrapher = async (
+ knex: db.KnexReadWriteTransaction,
+ {
+ user,
+ newConfig,
+ existingConfig,
+ shouldInherit,
+ referencedVariablesMightChange = true,
+ }: {
+ user: DbPlainUser
+ newConfig: GrapherInterface
+ existingConfig?: GrapherInterface
+ // if undefined, keep inheritance as is.
+ // if true or false, enable or disable inheritance
+ shouldInherit?: boolean
+ // if the variables a chart uses can change then we need
+ // to update the latest country data which takes quite a long time (hundreds of ms)
+ referencedVariablesMightChange?: boolean
+ }
+) => {
+ // Try to migrate the new config to the latest version
+ newConfig = migrateGrapherConfigToLatestVersion(newConfig)
+
+ // Slugs need some special logic to ensure public urls remain consistent whenever possible
+ async function isSlugUsedInRedirect() {
+ const rows = await db.knexRaw(
+ knex,
+ `SELECT * FROM chart_slug_redirects WHERE chart_id != ? AND slug = ?`,
+ // -1 is a placeholder ID that will never exist; but we cannot use NULL because
+ // in that case we would always get back an empty resultset
+ [existingConfig ? existingConfig.id : -1, newConfig.slug]
+ )
+ return rows.length > 0
+ }
+
+ async function isSlugUsedInOtherGrapher() {
+ const rows = await db.knexRaw>(
+ knex,
+ `-- sql
+ SELECT c.id
+ FROM charts c
+ JOIN chart_configs cc ON cc.id = c.configId
+ WHERE
+ c.id != ?
+ AND cc.full ->> "$.isPublished" = "true"
+ AND cc.slug = ?
+ `,
+ // -1 is a placeholder ID that will never exist; but we cannot use NULL because
+ // in that case we would always get back an empty resultset
+ [existingConfig ? existingConfig.id : -1, newConfig.slug]
+ )
+ return rows.length > 0
+ }
+
+ // When a chart is published, check for conflicts
+ if (newConfig.isPublished) {
+ if (!isValidSlug(newConfig.slug))
+ throw new JsonError(`Invalid chart slug ${newConfig.slug}`)
+ else if (await isSlugUsedInRedirect())
+ throw new JsonError(
+ `This chart slug was previously used by another chart: ${newConfig.slug}`
+ )
+ else if (await isSlugUsedInOtherGrapher())
+ throw new JsonError(
+ `This chart slug is in use by another published chart: ${newConfig.slug}`
+ )
+ else if (
+ existingConfig &&
+ existingConfig.isPublished &&
+ existingConfig.slug !== newConfig.slug
+ ) {
+ // Changing slug of an existing chart, delete any old redirect and create new one
+ await db.knexRaw(
+ knex,
+ `DELETE FROM chart_slug_redirects WHERE chart_id = ? AND slug = ?`,
+ [existingConfig.id, existingConfig.slug]
+ )
+ await db.knexRaw(
+ knex,
+ `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`,
+ [existingConfig.id, existingConfig.slug]
+ )
+ // When we rename grapher configs, make sure to delete the old one (the new one will be saved below)
+ await deleteGrapherConfigFromR2(
+ R2GrapherConfigDirectory.publishedGrapherBySlug,
+ `${existingConfig.slug}.json`
+ )
+ }
+ }
+
+ if (existingConfig)
+ // Bump chart version, very important for cachebusting
+ newConfig.version = existingConfig.version! + 1
+ else if (newConfig.version)
+ // If a chart is republished, we want to keep incrementing the old version number,
+ // otherwise it can lead to clients receiving cached versions of the old data.
+ newConfig.version += 1
+ else newConfig.version = 1
+
+ // add the isPublished field if is missing
+ if (newConfig.isPublished === undefined) {
+ newConfig.isPublished = false
+ }
+
+ // Execute the actual database update or creation
+ let chartId: number
+ let chartConfigId: Base64String
+ let patchConfig: GrapherInterface
+ let fullConfig: GrapherInterface
+ if (existingConfig) {
+ chartId = existingConfig.id!
+ const configs = await updateExistingChart(knex, {
+ config: newConfig,
+ user,
+ chartId,
+ shouldInherit,
+ })
+ chartConfigId = configs.chartConfigId
+ patchConfig = configs.patchConfig
+ fullConfig = configs.fullConfig
+ } else {
+ const configs = await saveNewChart(knex, {
+ config: newConfig,
+ user,
+ shouldInherit,
+ })
+ chartConfigId = configs.chartConfigId
+ patchConfig = configs.patchConfig
+ fullConfig = configs.fullConfig
+ chartId = fullConfig.id!
+ }
+
+ // Record this change in version history
+ const chartRevisionLog = {
+ chartId: chartId as number,
+ userId: user.id,
+ config: serializeChartConfig(patchConfig),
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ } satisfies DbInsertChartRevision
+ await db.knexRaw(
+ knex,
+ `INSERT INTO chart_revisions (chartId, userId, config, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`,
+ [
+ chartRevisionLog.chartId,
+ chartRevisionLog.userId,
+ chartRevisionLog.config,
+ chartRevisionLog.createdAt,
+ chartRevisionLog.updatedAt,
+ ]
+ )
+
+ // Remove any old dimensions and store the new ones
+ // We only note that a relationship exists between the chart and variable in the database; the actual dimension configuration is left to the json
+ await db.knexRaw(knex, `DELETE FROM chart_dimensions WHERE chartId=?`, [
+ chartId,
+ ])
+
+ const newDimensions = fullConfig.dimensions ?? []
+ for (const [i, dim] of newDimensions.entries()) {
+ await db.knexRaw(
+ knex,
+ `INSERT INTO chart_dimensions (chartId, variableId, property, \`order\`) VALUES (?, ?, ?, ?)`,
+ [chartId, dim.variableId, dim.property, i]
+ )
+ }
+
+ // So we can generate country profiles including this chart data
+ if (fullConfig.isPublished && referencedVariablesMightChange)
+ // TODO: remove this ad hoc knex transaction context when we switch the function to knex
+ await denormalizeLatestCountryData(
+ knex,
+ newDimensions.map((d) => d.variableId)
+ )
+
+ if (fullConfig.isPublished) {
+ await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId, {
+ directory: R2GrapherConfigDirectory.publishedGrapherBySlug,
+ filename: `${fullConfig.slug}.json`,
+ })
+ }
+
+ if (
+ fullConfig.isPublished &&
+ (!existingConfig || !existingConfig.isPublished)
+ ) {
+ // Newly published, set publication info
+ await db.knexRaw(
+ knex,
+ `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `,
+ [new Date(), user.id, chartId]
+ )
+ await triggerStaticBuild(user, `Publishing chart ${fullConfig.slug}`)
+ } else if (
+ !fullConfig.isPublished &&
+ existingConfig &&
+ existingConfig.isPublished
+ ) {
+ // Unpublishing chart, delete any existing redirects to it
+ await db.knexRaw(
+ knex,
+ `DELETE FROM chart_slug_redirects WHERE chart_id = ?`,
+ [existingConfig.id]
+ )
+ await deleteGrapherConfigFromR2(
+ R2GrapherConfigDirectory.publishedGrapherBySlug,
+ `${existingConfig.slug}.json`
+ )
+ await triggerStaticBuild(user, `Unpublishing chart ${fullConfig.slug}`)
+ } else if (fullConfig.isPublished)
+ await triggerStaticBuild(user, `Updating chart ${fullConfig.slug}`)
+
+ return {
+ chartId,
+ savedPatch: patchConfig,
+ }
+}
+
+export async function updateGrapherConfigsInR2(
+ knex: db.KnexReadonlyTransaction,
+ updatedCharts: { chartConfigId: string; isPublished: boolean }[],
+ updatedMultiDimViews: { chartConfigId: string; isPublished: boolean }[]
+) {
+ const idsToUpdate = [
+ ...updatedCharts.filter(({ isPublished }) => isPublished),
+ ...updatedMultiDimViews,
+ ].map(({ chartConfigId }) => chartConfigId)
+ const builder = knex(ChartConfigsTableName)
+ .select("id", "full", "fullMd5")
+ .whereIn("id", idsToUpdate)
+ for await (const { id, full, fullMd5 } of builder.stream()) {
+ await saveGrapherConfigToR2ByUUID(id, full, fullMd5)
+ }
+}
+
+export async function getChartsJson(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000
+ const charts = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT ${oldChartFieldList},
+ round(views_365d / 365, 1) as pageviewsPerDay
+ FROM charts
+ JOIN chart_configs ON chart_configs.id = charts.configId
+ JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
+ LEFT JOIN analytics_pageviews on (analytics_pageviews.url = CONCAT("https://ourworldindata.org/grapher/", chart_configs.slug) AND chart_configs.full ->> '$.isPublished' = "true" )
+ LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
+ ORDER BY charts.lastEditedAt DESC LIMIT ?
+ `,
+ [limit]
+ )
+
+ await assignTagsForCharts(trx, charts)
+
+ return { charts }
+}
+
+export async function getChartsCsv(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000
+
+ // note: this query is extended from OldChart.listFields.
+ const charts = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT
+ charts.id,
+ chart_configs.full->>"$.version" AS version,
+ CONCAT("${BAKED_BASE_URL}/grapher/", chart_configs.full->>"$.slug") AS url,
+ CONCAT("${ADMIN_BASE_URL}", "/admin/charts/", charts.id, "/edit") AS editUrl,
+ chart_configs.full->>"$.slug" AS slug,
+ chart_configs.full->>"$.title" AS title,
+ chart_configs.full->>"$.subtitle" AS subtitle,
+ chart_configs.full->>"$.sourceDesc" AS sourceDesc,
+ chart_configs.full->>"$.note" AS note,
+ chart_configs.chartType AS type,
+ chart_configs.full->>"$.internalNotes" AS internalNotes,
+ chart_configs.full->>"$.variantName" AS variantName,
+ chart_configs.full->>"$.isPublished" AS isPublished,
+ chart_configs.full->>"$.tab" AS tab,
+ chart_configs.chartType IS NOT NULL AS hasChartTab,
+ JSON_EXTRACT(chart_configs.full, "$.hasMapTab") = true AS hasMapTab,
+ chart_configs.full->>"$.originUrl" AS originUrl,
+ charts.lastEditedAt,
+ charts.lastEditedByUserId,
+ lastEditedByUser.fullName AS lastEditedBy,
+ charts.publishedAt,
+ charts.publishedByUserId,
+ publishedByUser.fullName AS publishedBy
+ FROM charts
+ JOIN chart_configs ON chart_configs.id = charts.configId
+ JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
+ LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
+ ORDER BY charts.lastEditedAt DESC
+ LIMIT ?
+ `,
+ [limit]
+ )
+ // note: retrieving references is VERY slow.
+ // await Promise.all(
+ // charts.map(async (chart: any) => {
+ // const references = await getReferencesByChartId(chart.id)
+ // chart.references = references.length
+ // ? references.map((ref) => ref.url)
+ // : ""
+ // })
+ // )
+ // await Chart.assignTagsForCharts(charts)
+ res.setHeader("Content-disposition", "attachment; filename=charts.csv")
+ res.setHeader("content-type", "text/csv")
+ const csv = Papa.unparse(charts)
+ return csv
+}
+
+export async function getChartConfigJson(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ return expectChartById(trx, req.params.chartId)
+}
+
+export async function getChartParentJson(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const chartId = expectInt(req.params.chartId)
+ const parent = await getParentByChartId(trx, chartId)
+ const isInheritanceEnabled = await isInheritanceEnabledForChart(
+ trx,
+ chartId
+ )
+ return omitUndefinedValues({
+ variableId: parent?.variableId,
+ config: parent?.config,
+ isActive: isInheritanceEnabled,
+ })
+}
+
+export async function getChartPatchConfigJson(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const chartId = expectInt(req.params.chartId)
+ const config = await expectPatchConfigByChartId(trx, chartId)
+ return config
+}
+
+export async function getChartLogsJson(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ return {
+ logs: await getLogsByChartId(
+ trx,
+ parseInt(req.params.chartId as string)
+ ),
+ }
+}
+
+export async function getChartReferencesJson(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const references = {
+ references: await getReferencesByChartId(
+ parseInt(req.params.chartId as string),
+ trx
+ ),
+ }
+ return references
+}
+
+export async function getChartRedirectsJson(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ return {
+ redirects: await getRedirectsByChartId(
+ trx,
+ parseInt(req.params.chartId as string)
+ ),
+ }
+}
+
+export async function getChartPageviewsJson(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const slug = await getChartSlugById(
+ trx,
+ parseInt(req.params.chartId as string)
+ )
+ if (!slug) return {}
+
+ const pageviewsByUrl = await db.knexRawFirst(
+ trx,
+ `-- sql
+ SELECT *
+ FROM
+ analytics_pageviews
+ WHERE
+ url = ?`,
+ [`https://ourworldindata.org/grapher/${slug}`]
+ )
+
+ return {
+ pageviews: pageviewsByUrl ?? undefined,
+ }
+}
+
+export async function createChart(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ let shouldInherit: boolean | undefined
+ if (req.query.inheritance) {
+ shouldInherit = req.query.inheritance === "enable"
+ }
+
+ try {
+ const { chartId } = await saveGrapher(trx, {
+ user: res.locals.user,
+ newConfig: req.body,
+ shouldInherit,
+ })
+
+ return { success: true, chartId: chartId }
+ } catch (err) {
+ return { success: false, error: String(err) }
+ }
+}
+
+export async function setChartTagsHandler(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const chartId = expectInt(req.params.chartId)
+
+ await setChartTags(trx, chartId, req.body.tags)
+
+ return { success: true }
+}
+
+export async function updateChart(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ let shouldInherit: boolean | undefined
+ if (req.query.inheritance) {
+ shouldInherit = req.query.inheritance === "enable"
+ }
+
+ const existingConfig = await expectChartById(trx, req.params.chartId)
+
+ try {
+ const { chartId, savedPatch } = await saveGrapher(trx, {
+ user: res.locals.user,
+ newConfig: req.body,
+ existingConfig,
+ shouldInherit,
+ })
+
+ const logs = await getLogsByChartId(trx, existingConfig.id as number)
+ return {
+ success: true,
+ chartId,
+ savedPatch,
+ newLog: logs[0],
+ }
+ } catch (err) {
+ return {
+ success: false,
+ error: String(err),
+ }
+ }
+}
+
+export async function deleteChart(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const chart = await expectChartById(trx, req.params.chartId)
+ if (chart.slug) {
+ const links = await getPublishedLinksTo(trx, [chart.slug])
+ if (links.length) {
+ const sources = links.map((link) => link.sourceSlug).join(", ")
+ throw new Error(
+ `Cannot delete chart in-use in the following published documents: ${sources}`
+ )
+ }
+ }
+
+ await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [
+ chart.id,
+ ])
+ await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE chart_id=?`, [
+ chart.id,
+ ])
+
+ const row = await db.knexRawFirst>(
+ trx,
+ `SELECT configId FROM charts WHERE id = ?`,
+ [chart.id]
+ )
+ if (!row || !row.configId)
+ throw new JsonError(`No chart config found for id ${chart.id}`, 404)
+ if (row) {
+ await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id])
+ await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [
+ row.configId,
+ ])
+ }
+
+ if (chart.isPublished)
+ await triggerStaticBuild(
+ res.locals.user,
+ `Deleting chart ${chart.slug}`
+ )
+
+ await deleteGrapherConfigFromR2ByUUID(row.configId)
+ if (chart.isPublished)
+ await deleteGrapherConfigFromR2(
+ R2GrapherConfigDirectory.publishedGrapherBySlug,
+ `${chart.slug}.json`
+ )
+
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/datasets.ts b/adminSiteServer/apiRoutes/datasets.ts
new file mode 100644
index 00000000000..fb490bc42ee
--- /dev/null
+++ b/adminSiteServer/apiRoutes/datasets.ts
@@ -0,0 +1,408 @@
+import {
+ DbPlainTag,
+ DbPlainDatasetTag,
+ JsonError,
+ DbRawVariable,
+ DbRawOrigin,
+ parseOriginsRow,
+} from "@ourworldindata/types"
+import {
+ OldChartFieldList,
+ oldChartFieldList,
+ assignTagsForCharts,
+} from "../../db/model/Chart.js"
+import { getDatasetById, setTagsForDataset } from "../../db/model/Dataset.js"
+import { logErrorAndMaybeSendToBugsnag } from "../../serverUtils/errorLog.js"
+import { expectInt } from "../../serverUtils/serverUtil.js"
+import {
+ syncDatasetToGitRepo,
+ removeDatasetFromGitRepo,
+} from "../gitDataExport.js"
+import { triggerStaticBuild } from "./routeUtils.js"
+import * as db from "../../db/db.js"
+import * as lodash from "lodash"
+import { Request } from "express"
+import * as e from "express"
+
+export async function getDatasets(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const datasets = await db.knexRaw>(
+ trx,
+ `-- sql
+ WITH variable_counts AS (
+ SELECT
+ v.datasetId,
+ COUNT(DISTINCT cd.chartId) as numCharts
+ FROM chart_dimensions cd
+ JOIN variables v ON cd.variableId = v.id
+ GROUP BY v.datasetId
+ )
+ SELECT
+ ad.id,
+ ad.namespace,
+ ad.name,
+ d.shortName,
+ ad.description,
+ ad.dataEditedAt,
+ du.fullName AS dataEditedByUserName,
+ ad.metadataEditedAt,
+ mu.fullName AS metadataEditedByUserName,
+ ad.isPrivate,
+ ad.nonRedistributable,
+ d.version,
+ vc.numCharts
+ FROM active_datasets ad
+ LEFT JOIN variable_counts vc ON ad.id = vc.datasetId
+ JOIN users du ON du.id=ad.dataEditedByUserId
+ JOIN users mu ON mu.id=ad.metadataEditedByUserId
+ JOIN datasets d ON d.id=ad.id
+ ORDER BY ad.dataEditedAt DESC
+ `
+ )
+
+ const tags = await db.knexRaw<
+ Pick & Pick
+ >(
+ trx,
+ `-- sql
+ SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt
+ JOIN tags t ON dt.tagId = t.id
+ `
+ )
+ const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId)
+ for (const dataset of datasets) {
+ dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) =>
+ lodash.omit(t, "datasetId")
+ )
+ }
+ /*LEFT JOIN variables AS v ON v.datasetId=d.id
+ GROUP BY d.id*/
+
+ return { datasets: datasets }
+}
+
+export async function getDataset(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const datasetId = expectInt(req.params.datasetId)
+
+ const dataset = await db.knexRawFirst>(
+ trx,
+ `-- sql
+ SELECT d.id,
+ d.namespace,
+ d.name,
+ d.shortName,
+ d.version,
+ d.description,
+ d.updatedAt,
+ d.dataEditedAt,
+ d.dataEditedByUserId,
+ du.fullName AS dataEditedByUserName,
+ d.metadataEditedAt,
+ d.metadataEditedByUserId,
+ mu.fullName AS metadataEditedByUserName,
+ d.isPrivate,
+ d.isArchived,
+ d.nonRedistributable,
+ d.updatePeriodDays
+ FROM datasets AS d
+ JOIN users du ON du.id=d.dataEditedByUserId
+ JOIN users mu ON mu.id=d.metadataEditedByUserId
+ WHERE d.id = ?
+ `,
+ [datasetId]
+ )
+
+ if (!dataset) throw new JsonError(`No dataset by id '${datasetId}'`, 404)
+
+ const zipFile = await db.knexRawFirst<{ filename: string }>(
+ trx,
+ `SELECT filename FROM dataset_files WHERE datasetId=?`,
+ [datasetId]
+ )
+ if (zipFile) dataset.zipFile = zipFile
+
+ const variables = await db.knexRaw<
+ Pick<
+ DbRawVariable,
+ "id" | "name" | "description" | "display" | "catalogPath"
+ >
+ >(
+ trx,
+ `-- sql
+ SELECT
+ v.id,
+ v.name,
+ v.description,
+ v.display,
+ v.catalogPath
+ FROM
+ variables AS v
+ WHERE
+ v.datasetId = ?
+ `,
+ [datasetId]
+ )
+
+ for (const v of variables) {
+ v.display = JSON.parse(v.display)
+ }
+
+ dataset.variables = variables
+
+ // add all origins
+ const origins: DbRawOrigin[] = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT DISTINCT
+ o.*
+ FROM
+ origins_variables AS ov
+ JOIN origins AS o ON ov.originId = o.id
+ JOIN variables AS v ON ov.variableId = v.id
+ WHERE
+ v.datasetId = ?
+ `,
+ [datasetId]
+ )
+
+ const parsedOrigins = origins.map(parseOriginsRow)
+
+ dataset.origins = parsedOrigins
+
+ const sources = await db.knexRaw<{
+ id: number
+ name: string
+ description: string
+ }>(
+ trx,
+ `
+ SELECT s.id, s.name, s.description
+ FROM sources AS s
+ WHERE s.datasetId = ?
+ ORDER BY s.id ASC
+ `,
+ [datasetId]
+ )
+
+ // expand description of sources and add to dataset as variableSources
+ dataset.variableSources = sources.map((s: any) => {
+ return {
+ id: s.id,
+ name: s.name,
+ ...JSON.parse(s.description),
+ }
+ })
+
+ const charts = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT ${oldChartFieldList}
+ FROM charts
+ JOIN chart_configs ON chart_configs.id = charts.configId
+ JOIN chart_dimensions AS cd ON cd.chartId = charts.id
+ JOIN variables AS v ON cd.variableId = v.id
+ JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
+ LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
+ WHERE v.datasetId = ?
+ GROUP BY charts.id
+ `,
+ [datasetId]
+ )
+
+ dataset.charts = charts
+
+ await assignTagsForCharts(trx, charts)
+
+ const tags = await db.knexRaw<{ id: number; name: string }>(
+ trx,
+ `
+ SELECT t.id, t.name
+ FROM tags t
+ JOIN dataset_tags dt ON dt.tagId = t.id
+ WHERE dt.datasetId = ?
+ `,
+ [datasetId]
+ )
+ dataset.tags = tags
+
+ const availableTags = await db.knexRaw<{
+ id: number
+ name: string
+ parentName: string
+ }>(
+ trx,
+ `
+ SELECT t.id, t.name, p.name AS parentName
+ FROM tags AS t
+ JOIN tags AS p ON t.parentId=p.id
+ `
+ )
+ dataset.availableTags = availableTags
+
+ return { dataset: dataset }
+}
+
+export async function updateDataset(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ // Only updates `nonRedistributable` and `tags`, other fields come from ETL
+ // and are not editable
+ const datasetId = expectInt(req.params.datasetId)
+ const dataset = await getDatasetById(trx, datasetId)
+ if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
+
+ const newDataset = (req.body as { dataset: any }).dataset
+ await db.knexRaw(
+ trx,
+ `
+ UPDATE datasets
+ SET
+ nonRedistributable=?,
+ metadataEditedAt=?,
+ metadataEditedByUserId=?
+ WHERE id=?
+ `,
+ [
+ newDataset.nonRedistributable,
+ new Date(),
+ _res.locals.user.id,
+ datasetId,
+ ]
+ )
+
+ const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId])
+ await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [
+ datasetId,
+ ])
+ if (tagRows.length)
+ for (const tagRow of tagRows) {
+ await db.knexRaw(
+ trx,
+ `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`,
+ tagRow
+ )
+ }
+
+ try {
+ await syncDatasetToGitRepo(trx, datasetId, {
+ oldDatasetName: dataset.name,
+ commitName: _res.locals.user.fullName,
+ commitEmail: _res.locals.user.email,
+ })
+ } catch (err) {
+ await logErrorAndMaybeSendToBugsnag(err, req)
+ // Continue
+ }
+
+ return { success: true }
+}
+
+export async function setArchived(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const datasetId = expectInt(req.params.datasetId)
+ const dataset = await getDatasetById(trx, datasetId)
+ if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
+
+ await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [
+ datasetId,
+ ])
+ return { success: true }
+}
+
+export async function setTags(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const datasetId = expectInt(req.params.datasetId)
+
+ await setTagsForDataset(trx, datasetId, req.body.tagIds)
+
+ return { success: true }
+}
+
+export async function deleteDataset(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const datasetId = expectInt(req.params.datasetId)
+
+ const dataset = await getDatasetById(trx, datasetId)
+ if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
+
+ await db.knexRaw(
+ trx,
+ `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`,
+ [datasetId]
+ )
+ await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [
+ datasetId,
+ ])
+ await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [
+ datasetId,
+ ])
+ await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [datasetId])
+ await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId])
+
+ try {
+ await removeDatasetFromGitRepo(dataset.name, dataset.namespace, {
+ commitName: _res.locals.user.fullName,
+ commitEmail: _res.locals.user.email,
+ })
+ } catch (err: any) {
+ await logErrorAndMaybeSendToBugsnag(err, req)
+ // Continue
+ }
+
+ return { success: true }
+}
+
+export async function republishCharts(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const datasetId = expectInt(req.params.datasetId)
+
+ const dataset = await getDatasetById(trx, datasetId)
+ if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
+
+ if (req.body.republish) {
+ await db.knexRaw(
+ trx,
+ `-- sql
+ UPDATE chart_configs cc
+ JOIN charts c ON c.configId = cc.id
+ SET
+ cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1),
+ cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1)
+ WHERE c.id IN (
+ SELECT DISTINCT chart_dimensions.chartId
+ FROM chart_dimensions
+ JOIN variables ON variables.id = chart_dimensions.variableId
+ WHERE variables.datasetId = ?
+ )`,
+ [datasetId]
+ )
+ }
+
+ await triggerStaticBuild(
+ _res.locals.user,
+ `Republishing all charts in dataset ${dataset.name} (${dataset.id})`
+ )
+
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/explorer.ts b/adminSiteServer/apiRoutes/explorer.ts
new file mode 100644
index 00000000000..44b9caf6308
--- /dev/null
+++ b/adminSiteServer/apiRoutes/explorer.ts
@@ -0,0 +1,33 @@
+import { JsonError } from "@ourworldindata/types"
+import { Request } from "express"
+import * as e from "express"
+
+import * as db from "../../db/db.js"
+export async function addExplorerTags(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const { slug } = req.params
+ const { tagIds } = req.body
+ const explorer = await trx.table("explorers").where({ slug }).first()
+ if (!explorer)
+ throw new JsonError(`No explorer found for slug ${slug}`, 404)
+
+ await trx.table("explorer_tags").where({ explorerSlug: slug }).delete()
+ for (const tagId of tagIds) {
+ await trx.table("explorer_tags").insert({ explorerSlug: slug, tagId })
+ }
+
+ return { success: true }
+}
+
+export async function deleteExplorerTags(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const { slug } = req.params
+ await trx.table("explorer_tags").where({ explorerSlug: slug }).delete()
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/gdocs.ts b/adminSiteServer/apiRoutes/gdocs.ts
new file mode 100644
index 00000000000..fbeb412e0d9
--- /dev/null
+++ b/adminSiteServer/apiRoutes/gdocs.ts
@@ -0,0 +1,289 @@
+import { getCanonicalUrl } from "@ourworldindata/components"
+import {
+ GdocsContentSource,
+ DbInsertUser,
+ JsonError,
+ GDOCS_BASE_URL,
+ gdocUrlRegex,
+ PostsGdocsLinksTableName,
+ PostsGdocsXImagesTableName,
+ PostsGdocsTableName,
+ PostsGdocsComponentsTableName,
+} from "@ourworldindata/types"
+import { checkIsGdocPostExcludingFragments } from "@ourworldindata/utils"
+import { isEmpty } from "lodash"
+import { match } from "ts-pattern"
+import {
+ checkHasChanges,
+ getPublishingAction,
+ GdocPublishingAction,
+ checkIsLightningUpdate,
+} from "../../adminSiteClient/gdocsDeploy.js"
+import {
+ indexIndividualGdocPost,
+ removeIndividualGdocPostFromIndex,
+} from "../../baker/algolia/utils/pages.js"
+import { GdocAbout } from "../../db/model/Gdoc/GdocAbout.js"
+import { GdocAuthor } from "../../db/model/Gdoc/GdocAuthor.js"
+import { getMinimalGdocPostsByIds } from "../../db/model/Gdoc/GdocBase.js"
+import { GdocDataInsight } from "../../db/model/Gdoc/GdocDataInsight.js"
+import {
+ getAllGdocIndexItemsOrderedByUpdatedAt,
+ getAndLoadGdocById,
+ updateGdocContentOnly,
+ createOrLoadGdocById,
+ gdocFromJSON,
+ addImagesToContentGraph,
+ setLinksForGdoc,
+ GdocLinkUpdateMode,
+ upsertGdoc,
+ getGdocBaseObjectById,
+ setTagsForGdoc,
+} from "../../db/model/Gdoc/GdocFactory.js"
+import { GdocHomepage } from "../../db/model/Gdoc/GdocHomepage.js"
+import { GdocPost } from "../../db/model/Gdoc/GdocPost.js"
+import { triggerStaticBuild, enqueueLightningChange } from "./routeUtils.js"
+import * as db from "../../db/db.js"
+import * as lodash from "lodash"
+import { Request } from "../authentication.js"
+import e from "express"
+
+export async function getAllGdocIndexItems(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ return getAllGdocIndexItemsOrderedByUpdatedAt(trx)
+}
+
+export async function getIndividualGdoc(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const id = req.params.id
+ const contentSource = req.query.contentSource as
+ | GdocsContentSource
+ | undefined
+
+ try {
+ // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published
+ const gdoc = await getAndLoadGdocById(trx, id, contentSource)
+
+ if (!gdoc.published) {
+ await updateGdocContentOnly(trx, id, gdoc)
+ }
+
+ res.set("Cache-Control", "no-store")
+ res.send(gdoc)
+ } catch (error) {
+ console.error("Error fetching gdoc", error)
+ res.status(500).json({
+ error: { message: String(error), status: 500 },
+ })
+ }
+}
+
+/**
+ * Handles all four `GdocPublishingAction` cases
+ * - SavingDraft (no action)
+ * - Publishing (index and bake)
+ * - Updating (index and bake (potentially via lightning deploy))
+ * - Unpublishing (remove from index and bake)
+ */
+async function indexAndBakeGdocIfNeccesary(
+ trx: db.KnexReadWriteTransaction,
+ user: Required,
+ prevGdoc:
+ | GdocPost
+ | GdocDataInsight
+ | GdocHomepage
+ | GdocAbout
+ | GdocAuthor,
+ nextGdoc: GdocPost | GdocDataInsight | GdocHomepage | GdocAbout | GdocAuthor
+) {
+ const prevJson = prevGdoc.toJSON()
+ const nextJson = nextGdoc.toJSON()
+ const hasChanges = checkHasChanges(prevGdoc, nextGdoc)
+ const action = getPublishingAction(prevJson, nextJson)
+ const isGdocPost = checkIsGdocPostExcludingFragments(nextJson)
+
+ await match(action)
+ .with(GdocPublishingAction.SavingDraft, lodash.noop)
+ .with(GdocPublishingAction.Publishing, async () => {
+ if (isGdocPost) {
+ await indexIndividualGdocPost(
+ nextJson,
+ trx,
+ // If the gdoc is being published for the first time, prevGdoc.slug will be undefined
+ // In that case, we pass nextJson.slug to see if it has any page views (i.e. from WP)
+ prevGdoc.slug || nextJson.slug
+ )
+ }
+ await triggerStaticBuild(user, `${action} ${nextJson.slug}`)
+ })
+ .with(GdocPublishingAction.Updating, async () => {
+ if (isGdocPost) {
+ await indexIndividualGdocPost(nextJson, trx, prevGdoc.slug)
+ }
+ if (checkIsLightningUpdate(prevJson, nextJson, hasChanges)) {
+ await enqueueLightningChange(
+ user,
+ `Lightning update ${nextJson.slug}`,
+ nextJson.slug
+ )
+ } else {
+ await triggerStaticBuild(user, `${action} ${nextJson.slug}`)
+ }
+ })
+ .with(GdocPublishingAction.Unpublishing, async () => {
+ if (isGdocPost) {
+ await removeIndividualGdocPostFromIndex(nextJson)
+ }
+ await triggerStaticBuild(user, `${action} ${nextJson.slug}`)
+ })
+ .exhaustive()
+}
+
+/**
+ * Only supports creating a new empty Gdoc or updating an existing one. Does not
+ * support creating a new Gdoc from an existing one. Relevant updates will
+ * trigger a deploy.
+ */
+export async function createOrUpdateGdoc(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const { id } = req.params
+
+ if (isEmpty(req.body)) {
+ return createOrLoadGdocById(trx, id)
+ }
+
+ const prevGdoc = await getAndLoadGdocById(trx, id)
+ if (!prevGdoc) throw new JsonError(`No Google Doc with id ${id} found`)
+
+ const nextGdoc = gdocFromJSON(req.body)
+ await nextGdoc.loadState(trx)
+
+ await addImagesToContentGraph(trx, nextGdoc)
+
+ await setLinksForGdoc(
+ trx,
+ nextGdoc.id,
+ nextGdoc.links,
+ nextGdoc.published
+ ? GdocLinkUpdateMode.DeleteAndInsert
+ : GdocLinkUpdateMode.DeleteOnly
+ )
+
+ await upsertGdoc(trx, nextGdoc)
+
+ await indexAndBakeGdocIfNeccesary(trx, res.locals.user, prevGdoc, nextGdoc)
+
+ return nextGdoc
+}
+
+async function validateTombstoneRelatedLinkUrl(
+ trx: db.KnexReadonlyTransaction,
+ relatedLink?: string
+) {
+ if (!relatedLink || !relatedLink.startsWith(GDOCS_BASE_URL)) return
+ const id = relatedLink.match(gdocUrlRegex)?.[1]
+ if (!id) {
+ throw new JsonError(`Invalid related link: ${relatedLink}`)
+ }
+ const [gdoc] = await getMinimalGdocPostsByIds(trx, [id])
+ if (!gdoc) {
+ throw new JsonError(`Google Doc with ID ${id} not found`)
+ }
+ if (!gdoc.published) {
+ throw new JsonError(`Google Doc with ID ${id} is not published`)
+ }
+}
+
+export async function deleteGdoc(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const { id } = req.params
+
+ const gdoc = await getGdocBaseObjectById(trx, id, false)
+ if (!gdoc) throw new JsonError(`No Google Doc with id ${id} found`)
+
+ const gdocSlug = getCanonicalUrl("", gdoc)
+ const { tombstone } = req.body
+
+ if (tombstone) {
+ await validateTombstoneRelatedLinkUrl(trx, tombstone.relatedLinkUrl)
+ const slug = gdocSlug.replace("/", "")
+ const { relatedLinkThumbnail } = tombstone
+ if (relatedLinkThumbnail) {
+ const thumbnailExists = await db.checkIsImageInDB(
+ trx,
+ relatedLinkThumbnail
+ )
+ if (!thumbnailExists) {
+ throw new JsonError(
+ `Image with filename "${relatedLinkThumbnail}" not found`
+ )
+ }
+ }
+ await trx
+ .table("posts_gdocs_tombstones")
+ .insert({ ...tombstone, gdocId: id, slug })
+ await trx
+ .table("redirects")
+ .insert({ source: gdocSlug, target: `/deleted${gdocSlug}` })
+ }
+
+ await trx
+ .table("posts")
+ .where({ gdocSuccessorId: gdoc.id })
+ .update({ gdocSuccessorId: null })
+
+ await trx.table(PostsGdocsLinksTableName).where({ sourceId: id }).delete()
+ await trx.table(PostsGdocsXImagesTableName).where({ gdocId: id }).delete()
+ await trx.table(PostsGdocsTableName).where({ id }).delete()
+ await trx
+ .table(PostsGdocsComponentsTableName)
+ .where({ gdocId: id })
+ .delete()
+ if (gdoc.published && checkIsGdocPostExcludingFragments(gdoc)) {
+ await removeIndividualGdocPostFromIndex(gdoc)
+ }
+ if (gdoc.published) {
+ if (!tombstone && gdocSlug && gdocSlug !== "/") {
+ // Assets have TTL of one week in Cloudflare. Add a redirect to make sure
+ // the page is no longer accessible.
+ // https://developers.cloudflare.com/pages/configuration/serving-pages/#asset-retention
+ console.log(`Creating redirect for "${gdocSlug}" to "/"`)
+ await db.knexRawInsert(
+ trx,
+ `INSERT INTO redirects (source, target, ttl)
+ VALUES (?, ?, DATE_ADD(NOW(), INTERVAL 8 DAY))`,
+ [gdocSlug, "/"]
+ )
+ }
+ await triggerStaticBuild(res.locals.user, `Deleting ${gdocSlug}`)
+ }
+ return {}
+}
+
+export async function setGdocTags(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const { gdocId } = req.params
+ const { tagIds } = req.body
+ const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({
+ id: id,
+ }))
+
+ await setTagsForGdoc(trx, gdocId, tagIdsAsObjects)
+
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/images.ts b/adminSiteServer/apiRoutes/images.ts
new file mode 100644
index 00000000000..7fc71c08e70
--- /dev/null
+++ b/adminSiteServer/apiRoutes/images.ts
@@ -0,0 +1,263 @@
+import { DbEnrichedImage, JsonError } from "@ourworldindata/types"
+import pMap from "p-map"
+import {
+ validateImagePayload,
+ processImageContent,
+ uploadToCloudflare,
+ deleteFromCloudflare,
+} from "../imagesHelpers.js"
+import { triggerStaticBuild } from "./routeUtils.js"
+import * as db from "../../db/db.js"
+import * as lodash from "lodash"
+
+import { Request } from "../authentication.js"
+import e from "express"
+export async function getImagesHandler(
+ _: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ try {
+ const images = await db.getCloudflareImages(trx)
+ res.set("Cache-Control", "no-store")
+ res.send({ images })
+ } catch (error) {
+ console.error("Error fetching images", error)
+ res.status(500).json({
+ error: { message: String(error), status: 500 },
+ })
+ }
+}
+
+export async function postImageHandler(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const { filename, type, content } = validateImagePayload(req.body)
+
+ const { asBlob, dimensions, hash } = await processImageContent(
+ content,
+ type
+ )
+
+ const collision = await trx("images")
+ .where({
+ hash,
+ replacedBy: null,
+ })
+ .first()
+
+ if (collision) {
+ return {
+ success: false,
+ error: `An image with this content already exists (filename: ${collision.filename})`,
+ }
+ }
+
+ const preexisting = await trx("images")
+ .where("filename", "=", filename)
+ .first()
+
+ if (preexisting) {
+ return {
+ success: false,
+ error: "An image with this filename already exists",
+ }
+ }
+
+ const cloudflareId = await uploadToCloudflare(filename, asBlob)
+
+ if (!cloudflareId) {
+ return {
+ success: false,
+ error: "Failed to upload image",
+ }
+ }
+
+ await trx("images").insert({
+ filename,
+ originalWidth: dimensions.width,
+ originalHeight: dimensions.height,
+ cloudflareId,
+ updatedAt: new Date().getTime(),
+ userId: res.locals.user.id,
+ hash,
+ })
+
+ const image = await db.getCloudflareImage(trx, filename)
+
+ return {
+ success: true,
+ image,
+ }
+}
+/**
+ * Similar to the POST route, but for updating an existing image.
+ * Creates a new image entry in the database and uploads the new image to Cloudflare.
+ * The old image is marked as replaced by the new image.
+ */
+export async function putImageHandler(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const { type, content } = validateImagePayload(req.body)
+ const { asBlob, dimensions, hash } = await processImageContent(
+ content,
+ type
+ )
+ const collision = await trx("images")
+ .where({
+ hash,
+ replacedBy: null,
+ })
+ .first()
+
+ if (collision) {
+ return {
+ success: false,
+ error: `An exact copy of this image already exists (filename: ${collision.filename})`,
+ }
+ }
+
+ const { id } = req.params
+
+ const image = await trx("images")
+ .where("id", "=", id)
+ .first()
+
+ if (!image) {
+ throw new JsonError(`No image found for id ${id}`, 404)
+ }
+
+ const originalCloudflareId = image.cloudflareId
+ const originalFilename = image.filename
+ const originalAltText = image.defaultAlt
+
+ if (!originalCloudflareId) {
+ throw new JsonError(
+ `Image with id ${id} has no associated Cloudflare image`,
+ 400
+ )
+ }
+
+ const newCloudflareId = await uploadToCloudflare(originalFilename, asBlob)
+
+ if (!newCloudflareId) {
+ throw new JsonError("Failed to upload image", 500)
+ }
+
+ const [newImageId] = await trx("images").insert({
+ filename: originalFilename,
+ originalWidth: dimensions.width,
+ originalHeight: dimensions.height,
+ cloudflareId: newCloudflareId,
+ updatedAt: new Date().getTime(),
+ userId: res.locals.user.id,
+ defaultAlt: originalAltText,
+ hash,
+ version: image.version + 1,
+ })
+
+ await trx("images").where("id", "=", id).update({
+ replacedBy: newImageId,
+ })
+
+ const updated = await db.getCloudflareImage(trx, originalFilename)
+
+ await triggerStaticBuild(
+ res.locals.user,
+ `Updating image "${originalFilename}"`
+ )
+
+ return {
+ success: true,
+ image: updated,
+ }
+}
+// Update alt text via patch
+export async function patchImageHandler(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const { id } = req.params
+
+ const image = await trx("images")
+ .where("id", "=", id)
+ .first()
+
+ if (!image) {
+ throw new JsonError(`No image found for id ${id}`, 404)
+ }
+
+ const patchableImageProperties = ["defaultAlt"] as const
+ const patch = lodash.pick(req.body, patchableImageProperties)
+
+ if (Object.keys(patch).length === 0) {
+ throw new JsonError("No patchable properties provided", 400)
+ }
+
+ await trx("images").where({ id }).update(patch)
+
+ const updated = await trx("images")
+ .where("id", "=", id)
+ .first()
+
+ return {
+ success: true,
+ image: updated,
+ }
+}
+
+export async function deleteImageHandler(
+ req: Request,
+ _: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const { id } = req.params
+
+ const image = await trx("images")
+ .where("id", "=", id)
+ .first()
+
+ if (!image) {
+ throw new JsonError(`No image found for id ${id}`, 404)
+ }
+ if (!image.cloudflareId) {
+ throw new JsonError(`Image does not have a cloudflare ID`, 400)
+ }
+
+ const replacementChain = await db.selectReplacementChainForImage(trx, id)
+
+ await pMap(
+ replacementChain,
+ async (image) => {
+ if (image.cloudflareId) {
+ await deleteFromCloudflare(image.cloudflareId)
+ }
+ },
+ { concurrency: 5 }
+ )
+
+ // There's an ON DELETE CASCADE which will delete the replacements
+ await trx("images").where({ id }).delete()
+
+ return {
+ success: true,
+ }
+}
+
+export async function getImageUsageHandler(
+ _: Request,
+ __: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const usage = await db.getImageUsage(trx)
+
+ return {
+ success: true,
+ usage,
+ }
+}
diff --git a/adminSiteServer/apiRoutes/mdims.ts b/adminSiteServer/apiRoutes/mdims.ts
new file mode 100644
index 00000000000..7880662928e
--- /dev/null
+++ b/adminSiteServer/apiRoutes/mdims.ts
@@ -0,0 +1,35 @@
+import { JsonError, MultiDimDataPageConfigRaw } from "@ourworldindata/types"
+import { isMultiDimDataPagePublished } from "../../db/model/MultiDimDataPage.js"
+import { isValidSlug } from "../../serverUtils/serverUtil.js"
+import {
+ FEATURE_FLAGS,
+ FeatureFlagFeature,
+} from "../../settings/clientSettings.js"
+import { createMultiDimConfig } from "../multiDim.js"
+import { triggerStaticBuild } from "./routeUtils.js"
+import { Request } from "../authentication.js"
+import * as db from "../../db/db.js"
+import e from "express"
+
+export async function handleMultiDimDataPageRequest(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const { slug } = req.params
+ if (!isValidSlug(slug)) {
+ throw new JsonError(`Invalid multi-dim slug ${slug}`)
+ }
+ const rawConfig = req.body as MultiDimDataPageConfigRaw
+ const id = await createMultiDimConfig(trx, slug, rawConfig)
+ if (
+ FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) &&
+ (await isMultiDimDataPagePublished(trx, slug))
+ ) {
+ await triggerStaticBuild(
+ res.locals.user,
+ `Publishing multidimensional chart ${slug}`
+ )
+ }
+ return { success: true, id }
+}
diff --git a/adminSiteServer/apiRoutes/misc.ts b/adminSiteServer/apiRoutes/misc.ts
new file mode 100644
index 00000000000..f31d6c2d6f5
--- /dev/null
+++ b/adminSiteServer/apiRoutes/misc.ts
@@ -0,0 +1,177 @@
+// Get an ArchieML output of all the work produced by an author. This includes
+// gdoc articles, gdoc modular/linear topic pages and wordpress modular topic
+// pages. Data insights are excluded. This is used to manually populate the
+// [.secondary] section of the {.research-and-writing} block of author pages
+
+import { DbRawPostGdoc, JsonError } from "@ourworldindata/types"
+
+import * as db from "../../db/db.js"
+import * as lodash from "lodash"
+import path from "path"
+import { expectInt } from "../../serverUtils/serverUtil.js"
+import { Request } from "../authentication.js"
+import e from "express"
+// using the alternate template, which highlights topics rather than articles.
+export async function fetchAllWork(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ type WordpressPageRecord = {
+ isWordpressPage: number
+ } & Record<
+ "slug" | "title" | "subtitle" | "thumbnail" | "authors" | "publishedAt",
+ string
+ >
+ type GdocRecord = Pick
+
+ const author = req.query.author
+ const gdocs = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT id, publishedAt
+ FROM posts_gdocs
+ WHERE JSON_CONTAINS(content->'$.authors', '"${author}"')
+ AND type NOT IN ("data-insight", "fragment")
+ AND published = 1
+ `
+ )
+
+ // type: page
+ const wpModularTopicPages = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT
+ wpApiSnapshot->>"$.slug" as slug,
+ wpApiSnapshot->>"$.title.rendered" as title,
+ wpApiSnapshot->>"$.excerpt.rendered" as subtitle,
+ TRUE as isWordpressPage,
+ wpApiSnapshot->>"$.authors_name" as authors,
+ wpApiSnapshot->>"$.featured_media_paths.medium_large" as thumbnail,
+ wpApiSnapshot->>"$.date" as publishedAt
+ FROM posts p
+ WHERE wpApiSnapshot->>"$.content" LIKE '%topic-page%'
+ AND JSON_CONTAINS(wpApiSnapshot->'$.authors_name', '"${author}"')
+ AND wpApiSnapshot->>"$.status" = 'publish'
+ AND NOT EXISTS (
+ SELECT 1 FROM posts_gdocs pg
+ WHERE pg.slug = p.slug
+ AND pg.content->>'$.type' LIKE '%topic-page'
+ )
+ `
+ )
+
+ const isWordpressPage = (
+ post: WordpressPageRecord | GdocRecord
+ ): post is WordpressPageRecord =>
+ (post as WordpressPageRecord).isWordpressPage === 1
+
+ function* generateProperty(key: string, value: string) {
+ yield `${key}: ${value}\n`
+ }
+
+ const sortByDateDesc = (
+ a: GdocRecord | WordpressPageRecord,
+ b: GdocRecord | WordpressPageRecord
+ ): number => {
+ if (!a.publishedAt || !b.publishedAt) return 0
+ return (
+ new Date(b.publishedAt).getTime() -
+ new Date(a.publishedAt).getTime()
+ )
+ }
+
+ function* generateAllWorkArchieMl() {
+ for (const post of [...gdocs, ...wpModularTopicPages].sort(
+ sortByDateDesc
+ )) {
+ if (isWordpressPage(post)) {
+ yield* generateProperty(
+ "url",
+ `https://ourworldindata.org/${post.slug}`
+ )
+ yield* generateProperty("title", post.title)
+ yield* generateProperty("subtitle", post.subtitle)
+ yield* generateProperty(
+ "authors",
+ JSON.parse(post.authors).join(", ")
+ )
+ const parsedPath = path.parse(post.thumbnail)
+ yield* generateProperty(
+ "filename",
+ // /app/uploads/2021/09/reducing-fertilizer-768x301.png -> reducing-fertilizer.png
+ path.format({
+ name: parsedPath.name.replace(/-\d+x\d+$/, ""),
+ ext: parsedPath.ext,
+ })
+ )
+ yield "\n"
+ } else {
+ // this is a gdoc
+ yield* generateProperty(
+ "url",
+ `https://docs.google.com/document/d/${post.id}/edit`
+ )
+ yield "\n"
+ }
+ }
+ }
+
+ res.type("text/plain")
+ return [...generateAllWorkArchieMl()].join("")
+}
+
+export async function fetchNamespaces(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const rows = await db.knexRaw<{
+ name: string
+ description?: string
+ isArchived: boolean
+ }>(
+ trx,
+ `SELECT DISTINCT
+ namespace AS name,
+ namespaces.description AS description,
+ namespaces.isArchived AS isArchived
+ FROM active_datasets
+ JOIN namespaces ON namespaces.name = active_datasets.namespace`
+ )
+
+ return {
+ namespaces: lodash
+ .sortBy(rows, (row) => row.description)
+ .map((namespace) => ({
+ ...namespace,
+ isArchived: !!namespace.isArchived,
+ })),
+ }
+}
+
+export async function fetchSourceById(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const sourceId = expectInt(req.params.sourceId)
+
+ const source = await db.knexRawFirst>(
+ trx,
+ `
+ SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace
+ FROM sources AS s
+ JOIN active_datasets AS d ON d.id=s.datasetId
+ WHERE s.id=?`,
+ [sourceId]
+ )
+ if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404)
+ source.variables = await db.knexRaw(
+ trx,
+ `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`,
+ [sourceId]
+ )
+
+ return { source: source }
+}
diff --git a/adminSiteServer/apiRoutes/posts.ts b/adminSiteServer/apiRoutes/posts.ts
new file mode 100644
index 00000000000..f34714fc30a
--- /dev/null
+++ b/adminSiteServer/apiRoutes/posts.ts
@@ -0,0 +1,215 @@
+import {
+ PostsTableName,
+ DbRawPost,
+ DbRawPostWithGdocPublishStatus,
+ JsonError,
+ OwidGdocPostInterface,
+ OwidGdocType,
+ PostsGdocsTableName,
+} from "@ourworldindata/types"
+import { camelCaseProperties } from "@ourworldindata/utils"
+import { createGdocAndInsertOwidGdocPostContent } from "../../db/model/Gdoc/archieToGdoc.js"
+import { upsertGdoc, setTagsForGdoc } from "../../db/model/Gdoc/GdocFactory.js"
+import { GdocPost } from "../../db/model/Gdoc/GdocPost.js"
+import { setTagsForPost, getTagsByPostId } from "../../db/model/Post.js"
+import { expectInt } from "../../serverUtils/serverUtil.js"
+import * as db from "../../db/db.js"
+import { Request } from "../authentication.js"
+import e from "express"
+export async function handleGetPostsJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const raw_rows = await db.knexRaw(
+ trx,
+ `-- sql
+ WITH
+ posts_tags_aggregated AS (
+ SELECT
+ post_id,
+ IF(
+ COUNT(tags.id) = 0,
+ JSON_ARRAY(),
+ JSON_ARRAYAGG(JSON_OBJECT("id", tags.id, "name", tags.name))
+ ) AS tags
+ FROM
+ post_tags
+ LEFT JOIN tags ON tags.id = post_tags.tag_id
+ GROUP BY
+ post_id
+ ),
+ post_gdoc_slug_successors AS (
+ SELECT
+ posts.id,
+ IF(
+ COUNT(gdocSlugSuccessor.id) = 0,
+ JSON_ARRAY(),
+ JSON_ARRAYAGG(
+ JSON_OBJECT("id", gdocSlugSuccessor.id, "published", gdocSlugSuccessor.published)
+ )
+ ) AS gdocSlugSuccessors
+ FROM
+ posts
+ LEFT JOIN posts_gdocs gdocSlugSuccessor ON gdocSlugSuccessor.slug = posts.slug
+ GROUP BY
+ posts.id
+ )
+ SELECT
+ posts.id AS id,
+ posts.title AS title,
+ posts.type AS TYPE,
+ posts.slug AS slug,
+ STATUS,
+ updated_at_in_wordpress,
+ posts.authors,
+ posts_tags_aggregated.tags AS tags,
+ gdocSuccessorId,
+ gdocSuccessor.published AS isGdocSuccessorPublished,
+ -- posts can either have explict successors via the gdocSuccessorId column
+ -- or implicit successors if a gdoc has been created that uses the same slug
+ -- as a Wp post (the gdoc one wins once it is published)
+ post_gdoc_slug_successors.gdocSlugSuccessors AS gdocSlugSuccessors
+ FROM
+ posts
+ LEFT JOIN post_gdoc_slug_successors ON post_gdoc_slug_successors.id = posts.id
+ LEFT JOIN posts_gdocs gdocSuccessor ON gdocSuccessor.id = posts.gdocSuccessorId
+ LEFT JOIN posts_tags_aggregated ON posts_tags_aggregated.post_id = posts.id
+ ORDER BY
+ updated_at_in_wordpress DESC`,
+ []
+ )
+ const rows = raw_rows.map((row: any) => ({
+ ...row,
+ tags: JSON.parse(row.tags),
+ isGdocSuccessorPublished: !!row.isGdocSuccessorPublished,
+ gdocSlugSuccessors: JSON.parse(row.gdocSlugSuccessors),
+ authors: JSON.parse(row.authors),
+ }))
+
+ return { posts: rows }
+}
+
+export async function handleSetTagsForPost(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const postId = expectInt(req.params.postId)
+ await setTagsForPost(trx, postId, req.body.tagIds)
+ return { success: true }
+}
+
+export async function handleGetPostById(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const postId = expectInt(req.params.postId)
+ const post = (await trx
+ .table(PostsTableName)
+ .where({ id: postId })
+ .select("*")
+ .first()) as DbRawPost | undefined
+ return camelCaseProperties({ ...post })
+}
+
+export async function handleCreateGdoc(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const postId = expectInt(req.params.postId)
+ const allowRecreate = !!req.body.allowRecreate
+ const post = (await trx
+ .table("posts_with_gdoc_publish_status")
+ .where({ id: postId })
+ .select("*")
+ .first()) as DbRawPostWithGdocPublishStatus | undefined
+
+ if (!post) throw new JsonError(`No post found for id ${postId}`, 404)
+ const existingGdocId = post.gdocSuccessorId
+ if (!allowRecreate && existingGdocId)
+ throw new JsonError("A gdoc already exists for this post", 400)
+ if (allowRecreate && existingGdocId && post.isGdocPublished) {
+ throw new JsonError(
+ "A gdoc already exists for this post and it is already published",
+ 400
+ )
+ }
+ if (post.archieml === null)
+ throw new JsonError(
+ `ArchieML was not present for post with id ${postId}`,
+ 500
+ )
+ const tagsByPostId = await getTagsByPostId(trx)
+ const tags = tagsByPostId.get(postId) || []
+ const archieMl = JSON.parse(
+ // Google Docs interprets ®ion in grapher URLS as ®ion
+ // So we escape them here
+ post.archieml.replaceAll("&", "&")
+ ) as OwidGdocPostInterface
+ const gdocId = await createGdocAndInsertOwidGdocPostContent(
+ archieMl.content,
+ post.gdocSuccessorId
+ )
+ // If we did not yet have a gdoc associated with this post, we need to register
+ // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise
+ // we don't need to make changes to the DB (only the gdoc regeneration was required)
+ if (!existingGdocId) {
+ post.gdocSuccessorId = gdocId
+ // This is not ideal - we are using knex for on thing and typeorm for another
+ // which means that we can't wrap this in a transaction. We should probably
+ // move posts to use typeorm as well or at least have a typeorm alternative for it
+ await trx
+ .table(PostsTableName)
+ .where({ id: postId })
+ .update("gdocSuccessorId", gdocId)
+
+ const gdoc = new GdocPost(gdocId)
+ gdoc.slug = post.slug
+ gdoc.content.title = post.title
+ gdoc.content.type = archieMl.content.type || OwidGdocType.Article
+ gdoc.published = false
+ gdoc.createdAt = new Date()
+ gdoc.publishedAt = post.published_at
+ await upsertGdoc(trx, gdoc)
+ await setTagsForGdoc(trx, gdocId, tags)
+ }
+ return { googleDocsId: gdocId }
+}
+
+export async function handleUnlinkGdoc(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const postId = expectInt(req.params.postId)
+ const post = (await trx
+ .table("posts_with_gdoc_publish_status")
+ .where({ id: postId })
+ .select("*")
+ .first()) as DbRawPostWithGdocPublishStatus | undefined
+
+ if (!post) throw new JsonError(`No post found for id ${postId}`, 404)
+ const existingGdocId = post.gdocSuccessorId
+ if (!existingGdocId)
+ throw new JsonError("No gdoc exists for this post", 400)
+ if (existingGdocId && post.isGdocPublished) {
+ throw new JsonError(
+ "The GDoc is already published - you can't unlink it",
+ 400
+ )
+ }
+ // This is not ideal - we are using knex for on thing and typeorm for another
+ // which means that we can't wrap this in a transaction. We should probably
+ // move posts to use typeorm as well or at least have a typeorm alternative for it
+ await trx
+ .table(PostsTableName)
+ .where({ id: postId })
+ .update("gdocSuccessorId", null)
+
+ await trx.table(PostsGdocsTableName).where({ id: existingGdocId }).delete()
+
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/redirects.ts b/adminSiteServer/apiRoutes/redirects.ts
new file mode 100644
index 00000000000..44452fe0263
--- /dev/null
+++ b/adminSiteServer/apiRoutes/redirects.ts
@@ -0,0 +1,147 @@
+import { DbPlainChartSlugRedirect, JsonError } from "@ourworldindata/types"
+import { getRedirects } from "../../baker/redirects.js"
+import {
+ redirectWithSourceExists,
+ getChainedRedirect,
+ getRedirectById,
+} from "../../db/model/Redirect.js"
+import { expectInt } from "../../serverUtils/serverUtil.js"
+import { triggerStaticBuild } from "./routeUtils.js"
+import * as db from "../../db/db.js"
+import { Request } from "../authentication.js"
+import e from "express"
+export async function handleGetSiteRedirects(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ return { redirects: await getRedirects(trx) }
+}
+
+export async function handlePostNewSiteRedirect(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const { source, target } = req.body
+ const sourceAsUrl = new URL(source, "https://ourworldindata.org")
+ if (sourceAsUrl.pathname === "/")
+ throw new JsonError("Cannot redirect from /", 400)
+ if (await redirectWithSourceExists(trx, source)) {
+ throw new JsonError(
+ `Redirect with source ${source} already exists`,
+ 400
+ )
+ }
+ const chainedRedirect = await getChainedRedirect(trx, source, target)
+ if (chainedRedirect) {
+ throw new JsonError(
+ "Creating this redirect would create a chain, redirect from " +
+ `${chainedRedirect.source} to ${chainedRedirect.target} ` +
+ "already exists. " +
+ (target === chainedRedirect.source
+ ? `Please create the redirect from ${source} to ` +
+ `${chainedRedirect.target} directly instead.`
+ : `Please delete the existing redirect and create a ` +
+ `new redirect from ${chainedRedirect.source} to ` +
+ `${target} instead.`),
+ 400
+ )
+ }
+ const { insertId: id } = await db.knexRawInsert(
+ trx,
+ `INSERT INTO redirects (source, target) VALUES (?, ?)`,
+ [source, target]
+ )
+ await triggerStaticBuild(
+ res.locals.user,
+ `Creating redirect id=${id} source=${source} target=${target}`
+ )
+ return { success: true, redirect: { id, source, target } }
+}
+
+export async function handleDeleteSiteRedirect(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const id = expectInt(req.params.id)
+ const redirect = await getRedirectById(trx, id)
+ if (!redirect) {
+ throw new JsonError(`No redirect found for id ${id}`, 404)
+ }
+ await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id])
+ await triggerStaticBuild(
+ res.locals.user,
+ `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}`
+ )
+ return { success: true }
+}
+
+export async function handleGetRedirects(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ return {
+ redirects: await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT
+ r.id,
+ r.slug,
+ r.chart_id as chartId,
+ chart_configs.slug AS chartSlug
+ FROM chart_slug_redirects AS r
+ JOIN charts ON charts.id = r.chart_id
+ JOIN chart_configs ON chart_configs.id = charts.configId
+ ORDER BY r.id DESC
+ `
+ ),
+ }
+}
+
+export async function handlePostNewChartRedirect(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const chartId = expectInt(req.params.chartId)
+ const fields = req.body as { slug: string }
+ const result = await db.knexRawInsert(
+ trx,
+ `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`,
+ [chartId, fields.slug]
+ )
+ const redirectId = result.insertId
+ const redirect = await db.knexRaw(
+ trx,
+ `SELECT * FROM chart_slug_redirects WHERE id = ?`,
+ [redirectId]
+ )
+ return { success: true, redirect: redirect }
+}
+
+export async function handleDeleteChartRedirect(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const id = expectInt(req.params.id)
+
+ const redirect = await db.knexRawFirst(
+ trx,
+ `SELECT * FROM chart_slug_redirects WHERE id = ?`,
+ [id]
+ )
+
+ if (!redirect) throw new JsonError(`No redirect found for id ${id}`, 404)
+
+ await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [id])
+ await triggerStaticBuild(
+ res.locals.user,
+ `Deleting redirect from ${redirect.slug}`
+ )
+
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/routeUtils.ts b/adminSiteServer/apiRoutes/routeUtils.ts
new file mode 100644
index 00000000000..0e647f290c0
--- /dev/null
+++ b/adminSiteServer/apiRoutes/routeUtils.ts
@@ -0,0 +1,44 @@
+import { DbPlainUser } from "@ourworldindata/types"
+import { DeployQueueServer } from "../../baker/DeployQueueServer.js"
+import { BAKE_ON_CHANGE } from "../../settings/serverSettings.js"
+
+// Call this to trigger build and deployment of static charts on change
+export const triggerStaticBuild = async (
+ user: DbPlainUser,
+ commitMessage: string
+) => {
+ if (!BAKE_ON_CHANGE) {
+ console.log(
+ "Not triggering static build because BAKE_ON_CHANGE is false"
+ )
+ return
+ }
+
+ return new DeployQueueServer().enqueueChange({
+ timeISOString: new Date().toISOString(),
+ authorName: user.fullName,
+ authorEmail: user.email,
+ message: commitMessage,
+ })
+}
+
+export const enqueueLightningChange = async (
+ user: DbPlainUser,
+ commitMessage: string,
+ slug: string
+) => {
+ if (!BAKE_ON_CHANGE) {
+ console.log(
+ "Not triggering static build because BAKE_ON_CHANGE is false"
+ )
+ return
+ }
+
+ return new DeployQueueServer().enqueueChange({
+ timeISOString: new Date().toISOString(),
+ authorName: user.fullName,
+ authorEmail: user.email,
+ message: commitMessage,
+ slug,
+ })
+}
diff --git a/adminSiteServer/apiRoutes/suggest.ts b/adminSiteServer/apiRoutes/suggest.ts
new file mode 100644
index 00000000000..4a294d43282
--- /dev/null
+++ b/adminSiteServer/apiRoutes/suggest.ts
@@ -0,0 +1,64 @@
+import {
+ DbChartTagJoin,
+ JsonError,
+ DbEnrichedImage,
+} from "@ourworldindata/types"
+import { parseIntOrUndefined } from "@ourworldindata/utils"
+import { getGptTopicSuggestions } from "../../db/model/Chart.js"
+import { CLOUDFLARE_IMAGES_URL } from "../../settings/clientSettings.js"
+import { fetchGptGeneratedAltText } from "../imagesHelpers.js"
+import * as db from "../../db/db.js"
+import e from "express"
+import { Request } from "../authentication.js"
+
+export async function suggestGptTopics(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+): Promise> {
+ const chartId = parseIntOrUndefined(req.params.chartId)
+ if (!chartId) throw new JsonError(`Invalid chart ID`, 400)
+
+ const topics = await getGptTopicSuggestions(trx, chartId)
+
+ if (!topics.length)
+ throw new JsonError(
+ `No GPT topic suggestions found for chart ${chartId}`,
+ 404
+ )
+
+ return {
+ topics,
+ }
+}
+
+export async function suggestGptAltText(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+): Promise<{
+ success: boolean
+ altText: string | null
+}> {
+ const imageId = parseIntOrUndefined(req.params.imageId)
+ if (!imageId) throw new JsonError(`Invalid image ID`, 400)
+ const image = await trx("images")
+ .where("id", imageId)
+ .first()
+ if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404)
+
+ const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public`
+ let altText: string | null = ""
+ try {
+ altText = await fetchGptGeneratedAltText(src)
+ } catch (error) {
+ console.error(`Error fetching GPT alt text for image ${imageId}`, error)
+ throw new JsonError(`Error fetching GPT alt text: ${error}`, 500)
+ }
+
+ if (!altText) {
+ throw new JsonError(`Unable to generate alt text for image`, 404)
+ }
+
+ return { success: true, altText }
+}
diff --git a/adminSiteServer/apiRoutes/tagGraph.ts b/adminSiteServer/apiRoutes/tagGraph.ts
new file mode 100644
index 00000000000..bafeec6d51d
--- /dev/null
+++ b/adminSiteServer/apiRoutes/tagGraph.ts
@@ -0,0 +1,62 @@
+import { JsonError, FlatTagGraph } from "@ourworldindata/types"
+import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils"
+import * as db from "../../db/db.js"
+import * as lodash from "lodash"
+import { Request } from "../authentication.js"
+import e from "express"
+
+export async function handleGetFlatTagGraph(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const flatTagGraph = await db.getFlatTagGraph(trx)
+ return flatTagGraph
+}
+
+export async function handlePostTagGraph(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const tagGraph = req.body?.tagGraph as unknown
+ if (!tagGraph) {
+ throw new JsonError("No tagGraph provided", 400)
+ }
+
+ function validateFlatTagGraph(
+ tagGraph: Record
+ ): tagGraph is FlatTagGraph {
+ if (lodash.isObject(tagGraph)) {
+ for (const [key, value] of Object.entries(tagGraph)) {
+ if (!lodash.isString(key) && isNaN(Number(key))) {
+ return false
+ }
+ if (!lodash.isArray(value)) {
+ return false
+ }
+ for (const tag of value) {
+ if (
+ !(
+ checkIsPlainObjectWithGuard(tag) &&
+ lodash.isNumber(tag.weight) &&
+ lodash.isNumber(tag.parentId) &&
+ lodash.isNumber(tag.childId)
+ )
+ ) {
+ return false
+ }
+ }
+ }
+ }
+
+ return true
+ }
+
+ const isValid = validateFlatTagGraph(tagGraph)
+ if (!isValid) {
+ throw new JsonError("Invalid tag graph provided", 400)
+ }
+ await db.updateTagGraph(trx, tagGraph)
+ res.send({ success: true })
+}
diff --git a/adminSiteServer/apiRoutes/tags.ts b/adminSiteServer/apiRoutes/tags.ts
new file mode 100644
index 00000000000..40bec68cec4
--- /dev/null
+++ b/adminSiteServer/apiRoutes/tags.ts
@@ -0,0 +1,261 @@
+import {
+ DbPlainTag,
+ DbPlainDataset,
+ DbRawPostGdoc,
+ JsonError,
+} from "@ourworldindata/types"
+import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils"
+import {
+ OldChartFieldList,
+ oldChartFieldList,
+ assignTagsForCharts,
+} from "../../db/model/Chart.js"
+import { expectInt } from "../../serverUtils/serverUtil.js"
+import { UNCATEGORIZED_TAG_ID } from "../../settings/serverSettings.js"
+import * as db from "../../db/db.js"
+import * as lodash from "lodash"
+import e from "express"
+import { Request } from "../authentication.js"
+
+export async function getTagById(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const tagId = expectInt(req.params.tagId) as number | null
+
+ // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff
+ // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag
+ // every time we create a new chart etcs
+ const uncategorized = tagId === UNCATEGORIZED_TAG_ID
+
+ // TODO: when we have types for our endpoints, make tag of that type instead of any
+ const tag: any = await db.knexRawFirst<
+ Pick<
+ DbPlainTag,
+ "id" | "name" | "specialType" | "updatedAt" | "parentId" | "slug"
+ >
+ >(
+ trx,
+ `-- sql
+ SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug
+ FROM tags t LEFT JOIN tags p ON t.parentId=p.id
+ WHERE t.id = ?
+ `,
+ [tagId]
+ )
+
+ // Datasets tagged with this tag
+ const datasets = await db.knexRaw<
+ Pick<
+ DbPlainDataset,
+ | "id"
+ | "namespace"
+ | "name"
+ | "description"
+ | "createdAt"
+ | "updatedAt"
+ | "dataEditedAt"
+ | "isPrivate"
+ | "nonRedistributable"
+ > & { dataEditedByUserName: string }
+ >(
+ trx,
+ `-- sql
+ SELECT
+ d.id,
+ d.namespace,
+ d.name,
+ d.description,
+ d.createdAt,
+ d.updatedAt,
+ d.dataEditedAt,
+ du.fullName AS dataEditedByUserName,
+ d.isPrivate,
+ d.nonRedistributable
+ FROM active_datasets d
+ JOIN users du ON du.id=d.dataEditedByUserId
+ LEFT JOIN dataset_tags dt ON dt.datasetId = d.id
+ WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"}
+ ORDER BY d.dataEditedAt DESC
+ `,
+ uncategorized ? [] : [tagId]
+ )
+ tag.datasets = datasets
+
+ // The other tags for those datasets
+ if (tag.datasets.length) {
+ if (uncategorized) {
+ for (const dataset of tag.datasets) dataset.tags = []
+ } else {
+ const datasetTags = await db.knexRaw<{
+ datasetId: number
+ id: number
+ name: string
+ }>(
+ trx,
+ `-- sql
+ SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt
+ JOIN tags t ON dt.tagId = t.id
+ WHERE dt.datasetId IN (?)
+ `,
+ [tag.datasets.map((d: any) => d.id)]
+ )
+ const tagsByDatasetId = lodash.groupBy(
+ datasetTags,
+ (t) => t.datasetId
+ )
+ for (const dataset of tag.datasets) {
+ dataset.tags = tagsByDatasetId[dataset.id].map((t) =>
+ lodash.omit(t, "datasetId")
+ )
+ }
+ }
+ }
+
+ // Charts using datasets under this tag
+ const charts = await db.knexRaw(
+ trx,
+ `-- sql
+ SELECT ${oldChartFieldList} FROM charts
+ JOIN chart_configs ON chart_configs.id = charts.configId
+ LEFT JOIN chart_tags ct ON ct.chartId=charts.id
+ JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
+ LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
+ WHERE ct.tagId ${tagId === UNCATEGORIZED_TAG_ID ? "IS NULL" : "= ?"}
+ GROUP BY charts.id
+ ORDER BY charts.updatedAt DESC
+ `,
+ uncategorized ? [] : [tagId]
+ )
+ tag.charts = charts
+
+ await assignTagsForCharts(trx, charts)
+
+ // Subcategories
+ const children = await db.knexRaw<{ id: number; name: string }>(
+ trx,
+ `-- sql
+ SELECT t.id, t.name FROM tags t
+ WHERE t.parentId = ?
+ `,
+ [tag.id]
+ )
+ tag.children = children
+
+ const possibleParents = await db.knexRaw<{ id: number; name: string }>(
+ trx,
+ `-- sql
+ SELECT t.id, t.name FROM tags t
+ WHERE t.parentId IS NULL
+ `
+ )
+ tag.possibleParents = possibleParents
+
+ return {
+ tag,
+ }
+}
+
+export async function updateTag(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const tagId = expectInt(req.params.tagId)
+ const tag = (req.body as { tag: any }).tag
+ await db.knexRaw(
+ trx,
+ `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`,
+ [tag.name, new Date(), tag.slug, tagId]
+ )
+ if (tag.slug) {
+ // See if there's a published gdoc with a matching slug.
+ // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index,
+ // where the page for the topic is just an article.
+ const gdoc = await db.knexRaw>(
+ trx,
+ `-- sql
+ SELECT slug FROM posts_gdocs pg
+ WHERE EXISTS (
+ SELECT 1
+ FROM posts_gdocs_x_tags gt
+ WHERE pg.id = gt.gdocId AND gt.tagId = ?
+ ) AND pg.published = TRUE AND pg.slug = ?`,
+ [tagId, tag.slug]
+ )
+ if (!gdoc.length) {
+ return {
+ success: true,
+ tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug.
+
+Are you sure you haven't made a typo?`,
+ }
+ }
+ }
+ return { success: true }
+}
+
+export async function createTag(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const tag = req.body
+ function validateTag(
+ tag: unknown
+ ): tag is { name: string; slug: string | null } {
+ return (
+ checkIsPlainObjectWithGuard(tag) &&
+ typeof tag.name === "string" &&
+ (tag.slug === null ||
+ (typeof tag.slug === "string" && tag.slug !== ""))
+ )
+ }
+ if (!validateTag(tag)) throw new JsonError("Invalid tag", 400)
+
+ const conflictingTag = await db.knexRawFirst<{
+ name: string
+ slug: string | null
+ }>(
+ trx,
+ `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`,
+ [tag.name, tag.slug]
+ )
+ if (conflictingTag)
+ throw new JsonError(
+ conflictingTag.name === tag.name
+ ? `Tag with name ${tag.name} already exists`
+ : `Tag with slug ${tag.slug} already exists`,
+ 400
+ )
+
+ const now = new Date()
+ const result = await db.knexRawInsert(
+ trx,
+ `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`,
+ // parentId will be deprecated soon once we migrate fully to the tag graph
+ [tag.name, tag.slug, now, now]
+ )
+ return { success: true, tagId: result.insertId }
+}
+
+export async function getAllTags(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ return { tags: await db.getMinimalTagsWithIsTopic(trx) }
+}
+
+export async function deleteTag(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const tagId = expectInt(req.params.tagId)
+
+ await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId])
+
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/users.ts b/adminSiteServer/apiRoutes/users.ts
new file mode 100644
index 00000000000..e232fd15a37
--- /dev/null
+++ b/adminSiteServer/apiRoutes/users.ts
@@ -0,0 +1,115 @@
+import { DbPlainUser, UsersTableName, JsonError } from "@ourworldindata/types"
+import { parseIntOrUndefined } from "@ourworldindata/utils"
+import { pick } from "lodash"
+import { getUserById, updateUser, insertUser } from "../../db/model/User.js"
+import { expectInt } from "../../serverUtils/serverUtil.js"
+import * as db from "../../db/db.js"
+import { Request } from "../authentication.js"
+import e from "express"
+export async function getUsers(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ return {
+ users: await trx
+ .select(
+ "id" satisfies keyof DbPlainUser,
+ "email" satisfies keyof DbPlainUser,
+ "fullName" satisfies keyof DbPlainUser,
+ "isActive" satisfies keyof DbPlainUser,
+ "isSuperuser" satisfies keyof DbPlainUser,
+ "createdAt" satisfies keyof DbPlainUser,
+ "updatedAt" satisfies keyof DbPlainUser,
+ "lastLogin" satisfies keyof DbPlainUser,
+ "lastSeen" satisfies keyof DbPlainUser
+ )
+ .from(UsersTableName)
+ .orderBy("lastSeen", "desc"),
+ }
+}
+
+export async function getUserByIdHandler(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const id = parseIntOrUndefined(req.params.userId)
+ if (!id) throw new JsonError("No user id given")
+ const user = await getUserById(trx, id)
+ return { user }
+}
+
+export async function deleteUser(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ if (!res.locals.user.isSuperuser)
+ throw new JsonError("Permission denied", 403)
+
+ const userId = expectInt(req.params.userId)
+ await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId])
+
+ return { success: true }
+}
+
+export async function updateUserHandler(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ if (!res.locals.user.isSuperuser)
+ throw new JsonError("Permission denied", 403)
+
+ const userId = parseIntOrUndefined(req.params.userId)
+ const user = userId !== undefined ? await getUserById(trx, userId) : null
+ if (!user) throw new JsonError("No such user", 404)
+
+ user.fullName = req.body.fullName
+ user.isActive = req.body.isActive
+
+ await updateUser(trx, userId!, pick(user, ["fullName", "isActive"]))
+
+ return { success: true }
+}
+
+export async function addUser(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ if (!res.locals.user.isSuperuser)
+ throw new JsonError("Permission denied", 403)
+
+ const { email, fullName } = req.body
+
+ await insertUser(trx, {
+ email,
+ fullName,
+ })
+
+ return { success: true }
+}
+
+export async function addImageToUser(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const userId = expectInt(req.params.userId)
+ const imageId = expectInt(req.params.imageId)
+ await trx("images").where({ id: imageId }).update({ userId })
+ return { success: true }
+}
+
+export async function removeUserImage(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const userId = expectInt(req.params.userId)
+ const imageId = expectInt(req.params.imageId)
+ await trx("images").where({ id: imageId, userId }).update({ userId: null })
+ return { success: true }
+}
diff --git a/adminSiteServer/apiRoutes/variables.ts b/adminSiteServer/apiRoutes/variables.ts
new file mode 100644
index 00000000000..d8f21c20ed3
--- /dev/null
+++ b/adminSiteServer/apiRoutes/variables.ts
@@ -0,0 +1,542 @@
+import {
+ getVariableDataRoute,
+ getVariableMetadataRoute,
+ migrateGrapherConfigToLatestVersion,
+} from "@ourworldindata/grapher"
+import {
+ DbRawVariable,
+ DbPlainDataset,
+ JsonError,
+ DbPlainChart,
+ DbRawChartConfig,
+ GrapherInterface,
+ OwidVariableWithSource,
+ parseChartConfig,
+} from "@ourworldindata/types"
+import {
+ fetchS3DataValuesByPath,
+ fetchS3MetadataByPath,
+ getAllChartsForIndicator,
+ getGrapherConfigsForVariable,
+ getMergedGrapherConfigForVariable,
+ searchVariables,
+ updateAllChartsThatInheritFromIndicator,
+ updateAllMultiDimViewsThatInheritFromIndicator,
+ updateGrapherConfigAdminOfVariable,
+ updateGrapherConfigETLOfVariable,
+} from "../../db/model/Variable.js"
+import { DATA_API_URL } from "../../settings/clientSettings.js"
+import * as db from "../../db/db.js"
+import {
+ getParentVariableIdFromChartConfig,
+ omit,
+ parseIntOrUndefined,
+} from "@ourworldindata/utils"
+import {
+ OldChartFieldList,
+ oldChartFieldList,
+ assignTagsForCharts,
+} from "../../db/model/Chart.js"
+import { updateExistingFullConfig } from "../../db/model/ChartConfigs.js"
+import { expectInt } from "../../serverUtils/serverUtil.js"
+import { triggerStaticBuild } from "./routeUtils.js"
+import * as lodash from "lodash"
+import { updateGrapherConfigsInR2 } from "./charts.js"
+import { Request } from "../authentication.js"
+import e from "express"
+
+export async function getEditorVariablesJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const datasets = []
+ const rows = await db.knexRaw<
+ Pick & {
+ datasetId: number
+ datasetName: string
+ datasetVersion: string
+ } & Pick<
+ DbPlainDataset,
+ "namespace" | "isPrivate" | "nonRedistributable"
+ >
+ >(
+ trx,
+ `-- sql
+ SELECT
+ v.name,
+ v.id,
+ d.id as datasetId,
+ d.name as datasetName,
+ d.version as datasetVersion,
+ d.namespace,
+ d.isPrivate,
+ d.nonRedistributable
+ FROM variables as v JOIN active_datasets as d ON v.datasetId = d.id
+ ORDER BY d.updatedAt DESC
+ `
+ )
+
+ let dataset:
+ | {
+ id: number
+ name: string
+ version: string
+ namespace: string
+ isPrivate: boolean
+ nonRedistributable: boolean
+ variables: { id: number; name: string }[]
+ }
+ | undefined
+ for (const row of rows) {
+ if (!dataset || row.datasetName !== dataset.name) {
+ if (dataset) datasets.push(dataset)
+
+ dataset = {
+ id: row.datasetId,
+ name: row.datasetName,
+ version: row.datasetVersion,
+ namespace: row.namespace,
+ isPrivate: !!row.isPrivate,
+ nonRedistributable: !!row.nonRedistributable,
+ variables: [],
+ }
+ }
+
+ dataset.variables.push({
+ id: row.id,
+ name: row.name ?? "",
+ })
+ }
+
+ if (dataset) datasets.push(dataset)
+
+ return { datasets: datasets }
+}
+
+export async function getVariableDataJson(
+ req: Request,
+ _res: e.Response>,
+ _trx: db.KnexReadonlyTransaction
+) {
+ const variableStr = req.params.variableStr as string
+ if (!variableStr) throw new JsonError("No variable id given")
+ if (variableStr.includes("+"))
+ throw new JsonError(
+ "Requesting multiple variables at the same time is no longer supported"
+ )
+ const variableId = parseInt(variableStr)
+ if (isNaN(variableId)) throw new JsonError("Invalid variable id")
+ return await fetchS3DataValuesByPath(
+ getVariableDataRoute(DATA_API_URL, variableId) + "?nocache"
+ )
+}
+
+export async function getVariableMetadataJson(
+ req: Request,
+ _res: e.Response>,
+ _trx: db.KnexReadonlyTransaction
+) {
+ const variableStr = req.params.variableStr as string
+ if (!variableStr) throw new JsonError("No variable id given")
+ if (variableStr.includes("+"))
+ throw new JsonError(
+ "Requesting multiple variables at the same time is no longer supported"
+ )
+ const variableId = parseInt(variableStr)
+ if (isNaN(variableId)) throw new JsonError("Invalid variable id")
+ return await fetchS3MetadataByPath(
+ getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache"
+ )
+}
+
+export async function getVariablesJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const limit = parseIntOrUndefined(req.query.limit as string) ?? 50
+ const query = req.query.search as string
+ return await searchVariables(query, limit, trx)
+}
+
+export async function getVariablesUsagesJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const query = `-- sql
+ SELECT
+ variableId,
+ COUNT(DISTINCT chartId) AS usageCount
+ FROM
+ chart_dimensions
+ GROUP BY
+ variableId
+ ORDER BY
+ usageCount DESC`
+
+ const rows = await db.knexRaw(trx, query)
+
+ return rows
+}
+
+export async function getVariablesGrapherConfigETLPatchConfigJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+ const variable = await getGrapherConfigsForVariable(trx, variableId)
+ if (!variable) {
+ throw new JsonError(`Variable with id ${variableId} not found`, 500)
+ }
+ return variable.etl?.patchConfig ?? {}
+}
+
+export async function getVariablesGrapherConfigAdminPatchConfigJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+ const variable = await getGrapherConfigsForVariable(trx, variableId)
+ if (!variable) {
+ throw new JsonError(`Variable with id ${variableId} not found`, 500)
+ }
+ return variable.admin?.patchConfig ?? {}
+}
+
+export async function getVariablesMergedGrapherConfigJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+ const config = await getMergedGrapherConfigForVariable(trx, variableId)
+ return config ?? {}
+}
+
+export async function getVariablesVariableIdJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+
+ const variable = await fetchS3MetadataByPath(
+ getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache"
+ )
+
+ // XXX: Patch shortName onto the end of catalogPath when it's missing,
+ // a temporary hack since our S3 metadata is out of date with our DB.
+ // See: https://github.com/owid/etl/issues/2135
+ if (variable.catalogPath && !variable.catalogPath.includes("#")) {
+ variable.catalogPath += `#${variable.shortName}`
+ }
+
+ const rawCharts = await db.knexRaw<
+ OldChartFieldList & {
+ isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"]
+ config: DbRawChartConfig["full"]
+ }
+ >(
+ trx,
+ `-- sql
+ SELECT ${oldChartFieldList}, charts.isInheritanceEnabled, chart_configs.full AS config
+ FROM charts
+ JOIN chart_configs ON chart_configs.id = charts.configId
+ JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId
+ LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId
+ JOIN chart_dimensions cd ON cd.chartId = charts.id
+ WHERE cd.variableId = ?
+ GROUP BY charts.id
+ `,
+ [variableId]
+ )
+
+ // check for parent indicators
+ const charts = rawCharts.map((chart) => {
+ const parentIndicatorId = getParentVariableIdFromChartConfig(
+ parseChartConfig(chart.config)
+ )
+ const hasParentIndicator = parentIndicatorId !== undefined
+ return omit({ ...chart, hasParentIndicator }, "config")
+ })
+
+ await assignTagsForCharts(trx, charts)
+
+ const variableWithConfigs = await getGrapherConfigsForVariable(
+ trx,
+ variableId
+ )
+ const grapherConfigETL = variableWithConfigs?.etl?.patchConfig
+ const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig
+ const mergedGrapherConfig =
+ variableWithConfigs?.admin?.fullConfig ??
+ variableWithConfigs?.etl?.fullConfig
+
+ // add the variable's display field to the merged grapher config
+ if (mergedGrapherConfig) {
+ const [varDims, otherDims] = lodash.partition(
+ mergedGrapherConfig.dimensions ?? [],
+ (dim) => dim.variableId === variableId
+ )
+ const varDimsWithDisplay = varDims.map((dim) => ({
+ display: variable.display,
+ ...dim,
+ }))
+ mergedGrapherConfig.dimensions = [...varDimsWithDisplay, ...otherDims]
+ }
+
+ const variableWithCharts: OwidVariableWithSource & {
+ charts: Record
+ grapherConfig: GrapherInterface | undefined
+ grapherConfigETL: GrapherInterface | undefined
+ grapherConfigAdmin: GrapherInterface | undefined
+ } = {
+ ...variable,
+ charts,
+ grapherConfig: mergedGrapherConfig,
+ grapherConfigETL,
+ grapherConfigAdmin,
+ }
+
+ return {
+ variable: variableWithCharts,
+ } /*, vardata: await getVariableData([variableId]) }*/
+}
+
+export async function putVariablesVariableIdGrapherConfigETL(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+
+ let validConfig: GrapherInterface
+ try {
+ validConfig = migrateGrapherConfigToLatestVersion(req.body)
+ } catch (err) {
+ return {
+ success: false,
+ error: String(err),
+ }
+ }
+
+ const variable = await getGrapherConfigsForVariable(trx, variableId)
+ if (!variable) {
+ throw new JsonError(`Variable with id ${variableId} not found`, 500)
+ }
+
+ const { savedPatch, updatedCharts, updatedMultiDimViews } =
+ await updateGrapherConfigETLOfVariable(trx, variable, validConfig)
+
+ await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews)
+ const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews]
+
+ if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) {
+ await triggerStaticBuild(
+ res.locals.user,
+ `Updating ETL config for variable ${variableId}`
+ )
+ }
+
+ return { success: true, savedPatch }
+}
+
+export async function deleteVariablesVariableIdGrapherConfigETL(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+
+ const variable = await getGrapherConfigsForVariable(trx, variableId)
+ if (!variable) {
+ throw new JsonError(`Variable with id ${variableId} not found`, 500)
+ }
+
+ // no-op if the variable doesn't have an ETL config
+ if (!variable.etl) return { success: true }
+
+ const now = new Date()
+
+ // remove reference in the variables table
+ await db.knexRaw(
+ trx,
+ `-- sql
+ UPDATE variables
+ SET grapherConfigIdETL = NULL
+ WHERE id = ?
+ `,
+ [variableId]
+ )
+
+ // delete row in the chart_configs table
+ await db.knexRaw(
+ trx,
+ `-- sql
+ DELETE FROM chart_configs
+ WHERE id = ?
+ `,
+ [variable.etl.configId]
+ )
+
+ // update admin config if there is one
+ if (variable.admin) {
+ await updateExistingFullConfig(trx, {
+ configId: variable.admin.configId,
+ config: variable.admin.patchConfig,
+ updatedAt: now,
+ })
+ }
+
+ const updates = {
+ patchConfigAdmin: variable.admin?.patchConfig,
+ updatedAt: now,
+ }
+ const updatedCharts = await updateAllChartsThatInheritFromIndicator(
+ trx,
+ variableId,
+ updates
+ )
+ const updatedMultiDimViews =
+ await updateAllMultiDimViewsThatInheritFromIndicator(
+ trx,
+ variableId,
+ updates
+ )
+ await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews)
+ const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews]
+
+ if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) {
+ await triggerStaticBuild(
+ res.locals.user,
+ `Updating ETL config for variable ${variableId}`
+ )
+ }
+
+ return { success: true }
+}
+
+export async function putVariablesVariableIdGrapherConfigAdmin(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+
+ let validConfig: GrapherInterface
+ try {
+ validConfig = migrateGrapherConfigToLatestVersion(req.body)
+ } catch (err) {
+ return {
+ success: false,
+ error: String(err),
+ }
+ }
+
+ const variable = await getGrapherConfigsForVariable(trx, variableId)
+ if (!variable) {
+ throw new JsonError(`Variable with id ${variableId} not found`, 500)
+ }
+
+ const { savedPatch, updatedCharts, updatedMultiDimViews } =
+ await updateGrapherConfigAdminOfVariable(trx, variable, validConfig)
+
+ await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews)
+ const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews]
+
+ if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) {
+ await triggerStaticBuild(
+ res.locals.user,
+ `Updating admin-authored config for variable ${variableId}`
+ )
+ }
+
+ return { success: true, savedPatch }
+}
+
+export async function deleteVariablesVariableIdGrapherConfigAdmin(
+ req: Request,
+ res: e.Response>,
+ trx: db.KnexReadWriteTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+
+ const variable = await getGrapherConfigsForVariable(trx, variableId)
+ if (!variable) {
+ throw new JsonError(`Variable with id ${variableId} not found`, 500)
+ }
+
+ // no-op if the variable doesn't have an admin-authored config
+ if (!variable.admin) return { success: true }
+
+ const now = new Date()
+
+ // remove reference in the variables table
+ await db.knexRaw(
+ trx,
+ `-- sql
+ UPDATE variables
+ SET grapherConfigIdAdmin = NULL
+ WHERE id = ?
+ `,
+ [variableId]
+ )
+
+ // delete row in the chart_configs table
+ await db.knexRaw(
+ trx,
+ `-- sql
+ DELETE FROM chart_configs
+ WHERE id = ?
+ `,
+ [variable.admin.configId]
+ )
+
+ const updates = {
+ patchConfigETL: variable.etl?.patchConfig,
+ updatedAt: now,
+ }
+ const updatedCharts = await updateAllChartsThatInheritFromIndicator(
+ trx,
+ variableId,
+ updates
+ )
+ const updatedMultiDimViews =
+ await updateAllMultiDimViewsThatInheritFromIndicator(
+ trx,
+ variableId,
+ updates
+ )
+ await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews)
+ const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews]
+
+ if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) {
+ await triggerStaticBuild(
+ res.locals.user,
+ `Updating admin-authored config for variable ${variableId}`
+ )
+ }
+
+ return { success: true }
+}
+
+export async function getVariablesVariableIdChartsJson(
+ req: Request,
+ _res: e.Response>,
+ trx: db.KnexReadonlyTransaction
+) {
+ const variableId = expectInt(req.params.variableId)
+ const charts = await getAllChartsForIndicator(trx, variableId)
+ return charts.map((chart) => ({
+ id: chart.chartId,
+ title: chart.config.title,
+ variantName: chart.config.variantName,
+ isChild: chart.isChild,
+ isInheritanceEnabled: chart.isInheritanceEnabled,
+ isPublished: chart.isPublished,
+ }))
+}
diff --git a/adminSiteServer/getLogsByChartId.ts b/adminSiteServer/getLogsByChartId.ts
new file mode 100644
index 00000000000..bbffc943807
--- /dev/null
+++ b/adminSiteServer/getLogsByChartId.ts
@@ -0,0 +1,34 @@
+import { Json } from "@ourworldindata/utils"
+import * as db from "../db/db.js"
+
+export async function getLogsByChartId(
+ knex: db.KnexReadonlyTransaction,
+ chartId: number
+): Promise<
+ {
+ userId: number
+ config: Json
+ userName: string
+ createdAt: Date
+ }[]
+> {
+ const logs = await db.knexRaw<{
+ userId: number
+ config: string
+ userName: string
+ createdAt: Date
+ }>(
+ knex,
+ `SELECT userId, config, fullName as userName, l.createdAt
+ FROM chart_revisions l
+ LEFT JOIN users u on u.id = userId
+ WHERE chartId = ?
+ ORDER BY l.id DESC
+ LIMIT 50`,
+ [chartId]
+ )
+ return logs.map((log) => ({
+ ...log,
+ config: JSON.parse(log.config),
+ }))
+}
diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx
index c422adcaa4e..52f9b4cb51a 100644
--- a/baker/SiteBaker.tsx
+++ b/baker/SiteBaker.tsx
@@ -56,6 +56,7 @@ import {
grabMetadataForGdocLinkedIndicator,
TombstonePageData,
gdocUrlRegex,
+ ChartViewInfo,
} from "@ourworldindata/utils"
import { execWrapper } from "../db/execWrapper.js"
import { countryProfileSpecs } from "../site/countryProfileProjects.js"
@@ -109,6 +110,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js"
import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js"
import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js"
import { getPublicDonorNames } from "../db/model/Donor.js"
+import { getChartViewsInfo } from "../db/model/ChartView.js"
type PrefetchedAttachments = {
donors: string[]
@@ -120,6 +122,7 @@ type PrefetchedAttachments = {
explorers: Record
}
linkedIndicators: Record
+ linkedChartViews: Record
}
// These aren't all "wordpress" steps
@@ -176,7 +179,7 @@ function getProgressBarTotal(bakeSteps: BakeStepConfig): number {
bakeSteps.has("dataInsights") ||
bakeSteps.has("authors")
) {
- total += 8
+ total += 9
}
return total
}
@@ -345,7 +348,7 @@ export class SiteBaker {
_prefetchedAttachmentsCache: PrefetchedAttachments | undefined = undefined
private async getPrefetchedGdocAttachments(
knex: db.KnexReadonlyTransaction,
- picks?: [string[], string[], string[], string[], string[]]
+ picks?: [string[], string[], string[], string[], string[], string[]]
): Promise {
if (!this._prefetchedAttachmentsCache) {
console.log("Prefetching attachments...")
@@ -459,6 +462,12 @@ export class SiteBaker {
name: `✅ Prefetched ${publishedAuthors.length} authors`,
})
+ const chartViewsInfo = await getChartViewsInfo(knex)
+ const chartViewsInfoByName = keyBy(chartViewsInfo, "name")
+ this.progressBar.tick({
+ name: `✅ Prefetched ${chartViewsInfo.length} chart views`,
+ })
+
const prefetchedAttachments = {
donors,
linkedAuthors: publishedAuthors,
@@ -469,6 +478,7 @@ export class SiteBaker {
graphers: publishedChartsBySlug,
},
linkedIndicators: datapageIndicatorsById,
+ linkedChartViews: chartViewsInfoByName,
}
this.progressBar.tick({ name: "✅ Prefetched attachments" })
this._prefetchedAttachmentsCache = prefetchedAttachments
@@ -480,6 +490,7 @@ export class SiteBaker {
imageFilenames,
linkedGrapherSlugs,
linkedExplorerSlugs,
+ linkedChartViewNames,
] = picks
const linkedDocuments = pick(
this._prefetchedAttachmentsCache.linkedDocuments,
@@ -528,6 +539,10 @@ export class SiteBaker {
this._prefetchedAttachmentsCache.linkedAuthors.filter(
(author) => authorNames.includes(author.name)
),
+ linkedChartViews: pick(
+ this._prefetchedAttachmentsCache.linkedChartViews,
+ linkedChartViewNames
+ ),
}
}
return this._prefetchedAttachmentsCache
@@ -619,6 +634,7 @@ export class SiteBaker {
publishedGdoc.linkedImageFilenames,
publishedGdoc.linkedChartSlugs.grapher,
publishedGdoc.linkedChartSlugs.explorer,
+ publishedGdoc.linkedChartViewNames,
])
publishedGdoc.donors = attachments.donors
publishedGdoc.linkedAuthors = attachments.linkedAuthors
@@ -629,6 +645,7 @@ export class SiteBaker {
...attachments.linkedCharts.explorers,
}
publishedGdoc.linkedIndicators = attachments.linkedIndicators
+ publishedGdoc.linkedChartViews = attachments.linkedChartViews
// this is a no-op if the gdoc doesn't have an all-chart block
if ("loadRelatedCharts" in publishedGdoc) {
@@ -876,6 +893,7 @@ export class SiteBaker {
dataInsight.linkedImageFilenames,
dataInsight.linkedChartSlugs.grapher,
dataInsight.linkedChartSlugs.explorer,
+ dataInsight.linkedChartViewNames,
])
dataInsight.linkedDocuments = attachments.linkedDocuments
dataInsight.imageMetadata = {
@@ -949,6 +967,7 @@ export class SiteBaker {
publishedAuthor.linkedImageFilenames,
publishedAuthor.linkedChartSlugs.grapher,
publishedAuthor.linkedChartSlugs.explorer,
+ publishedAuthor.linkedChartViewNames,
])
// We don't need these to be attached to the gdoc in the current
diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx
index e53a9e3e521..e64743f9324 100644
--- a/baker/siteRenderers.tsx
+++ b/baker/siteRenderers.tsx
@@ -441,6 +441,7 @@ ${dataInsights
latestDataInsights: get(post, "latestDataInsights", []),
homepageMetadata: get(post, "homepageMetadata", {}),
latestWorkLinks: get(post, "latestWorkLinks", []),
+ linkedChartViews: get(post, "linkedChartViews", {}),
}}
>
diff --git a/db/migrateWpPostsToArchieMl.ts b/db/migrateWpPostsToArchieMl.ts
index f9c35c0e84a..a0938d6c546 100644
--- a/db/migrateWpPostsToArchieMl.ts
+++ b/db/migrateWpPostsToArchieMl.ts
@@ -20,7 +20,7 @@ import {
adjustHeadingLevels,
findMinimumHeadingLevel,
} from "./model/Gdoc/htmlToEnriched.js"
-import { getPostRelatedCharts, isPostSlugCitable } from "./model/Post.js"
+import { getPostRelatedCharts } from "./model/Post.js"
import { enrichedBlocksToMarkdown } from "./model/Gdoc/enrichedToMarkdown.js"
// slugs from all the linear entries we want to migrate from @edomt
@@ -131,15 +131,6 @@ const migrate = async (trx: db.KnexReadWriteTransaction): Promise => {
relatedCharts = await getPostRelatedCharts(trx, post.id)
}
- const shouldIncludeMaxAsAuthor = isPostSlugCitable(post.slug)
- if (
- shouldIncludeMaxAsAuthor &&
- post.authors &&
- !post.authors.includes("Max Roser")
- ) {
- post.authors.push("Max Roser")
- }
-
// We don't get the first and last nodes if they are comments.
// This can cause issues with the wp:components so here we wrap
// everything in a div
diff --git a/db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts b/db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts
new file mode 100644
index 00000000000..06596eea89b
--- /dev/null
+++ b/db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts
@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class PostsGdocsLinksAddChartViews1734454799588
+ implements MigrationInterface
+{
+ public async up(queryRunner: QueryRunner): Promise {
+ await queryRunner.query(`
+ ALTER TABLE posts_gdocs_links
+ MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer', 'chart-view') NULL`)
+ }
+
+ public async down(queryRunner: QueryRunner): Promise {
+ await queryRunner.query(`
+ ALTER TABLE posts_gdocs_links
+ MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer') NULL`)
+ }
+}
diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts
new file mode 100644
index 00000000000..2bfd5ad8ed3
--- /dev/null
+++ b/db/model/ChartView.ts
@@ -0,0 +1,34 @@
+import { ChartViewInfo, JsonString } from "@ourworldindata/types"
+import * as db from "../db.js"
+
+export const getChartViewsInfo = async (
+ knex: db.KnexReadonlyTransaction,
+ names?: string[]
+): Promise => {
+ type RawRow = Omit & {
+ queryParamsForParentChart: JsonString
+ }
+ let rows: RawRow[]
+
+ const query = `-- sql
+SELECT cv.name,
+ cc.full ->> "$.title" as title,
+ chartConfigId,
+ pcc.slug as parentChartSlug,
+ cv.queryParamsForParentChart
+FROM chart_views cv
+JOIN chart_configs cc on cc.id = cv.chartConfigId
+JOIN charts pc on cv.parentChartId = pc.id
+JOIN chart_configs pcc on pc.configId = pcc.id
+ `
+
+ if (names) {
+ if (names.length === 0) return []
+ rows = await db.knexRaw(knex, `${query} WHERE cv.name IN (?)`, [names])
+ } else rows = await db.knexRaw(knex, query)
+
+ return rows.map((row) => ({
+ ...row,
+ queryParamsForParentChart: JSON.parse(row.queryParamsForParentChart),
+ }))
+}
diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts
index 754471c247a..173fd06e6fb 100644
--- a/db/model/Gdoc/GdocBase.ts
+++ b/db/model/Gdoc/GdocBase.ts
@@ -48,7 +48,7 @@ import {
getVariableMetadata,
getVariableOfDatapageIfApplicable,
} from "../Variable.js"
-import { createLinkFromUrl } from "../Link.js"
+import { createLinkForChartView, createLinkFromUrl } from "../Link.js"
import {
getMultiDimDataPageBySlug,
isMultiDimDataPagePublished,
@@ -56,6 +56,7 @@ import {
import {
ARCHVED_THUMBNAIL_FILENAME,
ChartConfigType,
+ ChartViewInfo,
DEFAULT_THUMBNAIL_FILENAME,
GrapherInterface,
LatestDataInsight,
@@ -66,6 +67,7 @@ import {
OwidGdocLinkType,
OwidGdocType,
} from "@ourworldindata/types"
+import { getChartViewsInfo } from "../ChartView.js"
export class GdocBase implements OwidGdocBaseInterface {
id!: string
@@ -89,6 +91,7 @@ export class GdocBase implements OwidGdocBaseInterface {
linkedIndicators: Record = {}
linkedDocuments: Record = {}
latestDataInsights: LatestDataInsight[] = []
+ linkedChartViews?: Record = {}
_omittableFields: string[] = []
constructor(id?: string) {
@@ -292,6 +295,14 @@ export class GdocBase implements OwidGdocBaseInterface {
return { grapher: [...grapher], explorer: [...explorer] }
}
+ get linkedChartViewNames(): string[] {
+ const filteredLinks = this.links
+ .filter((link) => link.linkType === OwidGdocLinkType.ChartView)
+ .map((link) => link.target)
+
+ return filteredLinks
+ }
+
get hasAllChartsBlock(): boolean {
let hasAllChartsBlock = false
for (const enrichedBlockSource of this.enrichedBlockSources) {
@@ -349,6 +360,13 @@ export class GdocBase implements OwidGdocBaseInterface {
componentType: block.type,
}),
])
+ .with({ type: "narrative-chart" }, (block) => [
+ createLinkForChartView({
+ name: block.name,
+ source: this,
+ componentType: block.type,
+ }),
+ ])
.with({ type: "all-charts" }, (block) =>
block.top.map((item) =>
createLinkFromUrl({
@@ -710,6 +728,11 @@ export class GdocBase implements OwidGdocBaseInterface {
}
}
+ async loadChartViewsInfo(knex: db.KnexReadonlyTransaction): Promise {
+ const result = await getChartViewsInfo(knex, this.linkedChartViewNames)
+ this.linkedChartViews = keyBy(result, "name")
+ }
+
async fetchAndEnrichGdoc(): Promise {
const docsClient = google.docs({
version: "v1",
@@ -855,6 +878,7 @@ export class GdocBase implements OwidGdocBaseInterface {
await this.loadImageMetadataFromDB(knex)
await this.loadLinkedCharts(knex)
await this.loadLinkedIndicators() // depends on linked charts
+ await this.loadChartViewsInfo(knex)
await this._loadSubclassAttachments(knex)
await this.validate(knex)
}
diff --git a/db/model/Gdoc/enrichedToMarkdown.ts b/db/model/Gdoc/enrichedToMarkdown.ts
index 2556f74ef2b..794700f30a3 100644
--- a/db/model/Gdoc/enrichedToMarkdown.ts
+++ b/db/model/Gdoc/enrichedToMarkdown.ts
@@ -127,6 +127,17 @@ ${items}
exportComponents
)
)
+ .with({ type: "narrative-chart" }, (b): string | undefined =>
+ markdownComponent(
+ "NarrativeChart",
+ {
+ name: b.name,
+ caption: b.caption ? spansToMarkdown(b.caption) : undefined,
+ // Note: truncated
+ },
+ exportComponents
+ )
+ )
.with({ type: "code" }, (b): string | undefined => {
return (
"```\n" +
diff --git a/db/model/Gdoc/enrichedToRaw.ts b/db/model/Gdoc/enrichedToRaw.ts
index bc27e3356ed..08e8e1fa288 100644
--- a/db/model/Gdoc/enrichedToRaw.ts
+++ b/db/model/Gdoc/enrichedToRaw.ts
@@ -48,6 +48,7 @@ import {
RawBlockPeople,
RawBlockPeopleRows,
RawBlockPerson,
+ RawBlockNarrativeChart,
RawBlockCode,
} from "@ourworldindata/types"
import { spanToHtmlString } from "./gdocUtils.js"
@@ -123,6 +124,20 @@ export function enrichedBlockToRawBlock(
},
})
)
+ .with(
+ { type: "narrative-chart" },
+ (b): RawBlockNarrativeChart => ({
+ type: b.type,
+ value: {
+ name: b.name,
+ height: b.height,
+ row: b.row,
+ column: b.column,
+ position: b.position,
+ caption: b.caption ? spansToHtmlText(b.caption) : undefined,
+ },
+ })
+ )
.with(
{ type: "code" },
(b): RawBlockCode => ({
diff --git a/db/model/Gdoc/exampleEnrichedBlocks.ts b/db/model/Gdoc/exampleEnrichedBlocks.ts
index 7e37a16bc27..74b2a0f8842 100644
--- a/db/model/Gdoc/exampleEnrichedBlocks.ts
+++ b/db/model/Gdoc/exampleEnrichedBlocks.ts
@@ -121,6 +121,16 @@ export const enrichedBlockExamples: Record<
caption: boldLinkExampleText,
parseErrors: [],
},
+ "narrative-chart": {
+ type: "narrative-chart",
+ name: "world-has-become-less-democratic",
+ height: "400",
+ row: "1",
+ column: "1",
+ position: "featured",
+ caption: boldLinkExampleText,
+ parseErrors: [],
+ },
code: {
type: "code",
text: [
diff --git a/db/model/Gdoc/extractGdocComponentInfo.ts b/db/model/Gdoc/extractGdocComponentInfo.ts
index 66b4c3c608a..308f06953c6 100644
--- a/db/model/Gdoc/extractGdocComponentInfo.ts
+++ b/db/model/Gdoc/extractGdocComponentInfo.ts
@@ -327,6 +327,7 @@ export function enumerateGdocComponentsWithoutChildren(
type: P.union(
"chart-story",
"chart",
+ "narrative-chart",
"horizontal-rule",
"html",
"image",
@@ -353,8 +354,8 @@ export function enumerateGdocComponentsWithoutChildren(
"additional-charts",
"simple-text",
"donors",
- "socials"
- // "narrative-chart" should go here once it's done
+ "socials",
+ "narrative-chart"
),
},
(c) => handleComponent(c, [], parentPath, path)
diff --git a/db/model/Gdoc/gdocUtils.ts b/db/model/Gdoc/gdocUtils.ts
index f08dfd85786..d53f3f41a5d 100644
--- a/db/model/Gdoc/gdocUtils.ts
+++ b/db/model/Gdoc/gdocUtils.ts
@@ -237,6 +237,7 @@ export function extractFilenamesFromBlock(
"latest-data-insights",
"list",
"missing-data",
+ "narrative-chart",
"numbered-list",
"people",
"people-rows",
diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts
index c2c9b50803d..b1b344a4491 100644
--- a/db/model/Gdoc/rawToArchie.ts
+++ b/db/model/Gdoc/rawToArchie.ts
@@ -47,6 +47,7 @@ import {
RawBlockPeople,
RawBlockPeopleRows,
RawBlockPerson,
+ RawBlockNarrativeChart,
RawBlockCode,
} from "@ourworldindata/types"
import { isArray } from "@ourworldindata/utils"
@@ -128,6 +129,21 @@ function* rawBlockChartToArchieMLString(
yield "{}"
}
+function* rawBlockNarrativeChartToArchieMLString(
+ block: RawBlockNarrativeChart
+): Generator {
+ yield "{.narrative-chart}"
+ if (typeof block.value !== "string") {
+ yield* propertyToArchieMLString("name", block.value)
+ yield* propertyToArchieMLString("height", block.value)
+ yield* propertyToArchieMLString("row", block.value)
+ yield* propertyToArchieMLString("column", block.value)
+ yield* propertyToArchieMLString("position", block.value)
+ yield* propertyToArchieMLString("caption", block.value)
+ }
+ yield "{}"
+}
+
function* rawBlockCodeToArchieMLString(
block: RawBlockCode
): Generator {
@@ -840,6 +856,10 @@ export function* OwidRawGdocBlockToArchieMLStringGenerator(
.with({ type: "all-charts" }, rawBlockAllChartsToArchieMLString)
.with({ type: "aside" }, rawBlockAsideToArchieMLString)
.with({ type: "chart" }, rawBlockChartToArchieMLString)
+ .with(
+ { type: "narrative-chart" },
+ rawBlockNarrativeChartToArchieMLString
+ )
.with({ type: "code" }, rawBlockCodeToArchieMLString)
.with({ type: "donors" }, rawBlockDonorListToArchieMLString)
.with({ type: "scroller" }, rawBlockScrollerToArchieMLString)
diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts
index a60bd1f4e79..209fcc59847 100644
--- a/db/model/Gdoc/rawToEnriched.ts
+++ b/db/model/Gdoc/rawToEnriched.ts
@@ -129,6 +129,8 @@ import {
EnrichedBlockPerson,
RawBlockPeopleRows,
EnrichedBlockPeopleRows,
+ RawBlockNarrativeChart,
+ EnrichedBlockNarrativeChart,
RawBlockCode,
EnrichedBlockCode,
} from "@ourworldindata/types"
@@ -172,6 +174,7 @@ export function parseRawBlocksToEnrichedBlocks(
.with({ type: "blockquote" }, parseBlockquote)
.with({ type: "callout" }, parseCallout)
.with({ type: "chart" }, parseChart)
+ .with({ type: "narrative-chart" }, parseNarrativeChart)
.with({ type: "code" }, parseCode)
.with({ type: "donors" }, parseDonorList)
.with({ type: "scroller" }, parseScroller)
@@ -496,6 +499,67 @@ const parseChart = (raw: RawBlockChart): EnrichedBlockChart => {
}
}
+const parseNarrativeChart = (
+ raw: RawBlockNarrativeChart
+): EnrichedBlockNarrativeChart => {
+ const createError = (
+ error: ParseError,
+ name: string,
+ caption: Span[] = []
+ ): EnrichedBlockNarrativeChart => ({
+ type: "narrative-chart",
+ name,
+ caption,
+ parseErrors: [error],
+ })
+
+ const val = raw.value
+
+ if (typeof val === "string") {
+ return {
+ type: "narrative-chart",
+ name: val,
+ parseErrors: [],
+ }
+ } else {
+ if (!val.name)
+ return createError(
+ {
+ message: "name property is missing",
+ },
+ ""
+ )
+
+ const warnings: ParseError[] = []
+
+ const height = val.height
+ const row = val.row
+ const column = val.column
+ // This property is currently unused, a holdover from @mathisonian's gdocs demo.
+ // We will decide soon™️ if we want to use it for something
+ let position: ChartPositionChoice | undefined = undefined
+ if (val.position)
+ if (val.position === "featured") position = val.position
+ else {
+ warnings.push({
+ message: "position must be 'featured' or unset",
+ })
+ }
+ const caption = val.caption ? htmlToSpans(val.caption) : []
+
+ return omitUndefinedValues({
+ type: "narrative-chart",
+ name: val.name,
+ height,
+ row,
+ column,
+ position,
+ caption: caption.length > 0 ? caption : undefined,
+ parseErrors: [],
+ }) as EnrichedBlockNarrativeChart
+ }
+}
+
const parseCode = (raw: RawBlockCode): EnrichedBlockCode => {
return {
type: "code",
diff --git a/db/model/Link.ts b/db/model/Link.ts
index 4468e6832dd..bb0d2941cf4 100644
--- a/db/model/Link.ts
+++ b/db/model/Link.ts
@@ -62,3 +62,23 @@ export function createLinkFromUrl({
sourceId: source.id,
} satisfies DbInsertPostGdocLink
}
+
+export function createLinkForChartView({
+ name,
+ source,
+ componentType,
+}: {
+ name: string
+ source: GdocBase
+ componentType: string
+}): DbInsertPostGdocLink {
+ return {
+ target: name,
+ linkType: OwidGdocLinkType.ChartView,
+ queryString: "",
+ hash: "",
+ text: "",
+ componentType,
+ sourceId: source.id,
+ } satisfies DbInsertPostGdocLink
+}
diff --git a/devTools/svgTester/update-configs.sh b/devTools/svgTester/update-configs.sh
index 6604fc59dc4..95298b748da 100755
--- a/devTools/svgTester/update-configs.sh
+++ b/devTools/svgTester/update-configs.sh
@@ -20,29 +20,49 @@ Make sure to run \`make refresh\` and \`make refresh.pageviews\` before running
main() {
echo "=> Resetting owid-grapher-svgs to origin/master"
- cd $SVGS_REPO\
- && git fetch\
- && git checkout -f master\
- && git reset --hard origin/master\
- && git clean -fd\
+ cd $SVGS_REPO \
+ && git fetch \
+ && git checkout -f master \
+ && git reset --hard origin/master \
+ && git clean -fdx \
&& cd -
- echo "=> Removing existing configs and reference svgs"
- rm -rf $CONFIGS_DIR $REFERENCES_DIR $ALL_VIEWS_DIR
-
echo "=> Dumping new configs and data"
+ rm -rf $CONFIGS_DIR
node itsJustJavascript/devTools/svgTester/dump-data.js -o $CONFIGS_DIR
node itsJustJavascript/devTools/svgTester/dump-chart-ids.js -o $CHART_IDS_FILE
- echo "=> Generating reference SVGs"
- node itsJustJavascript/devTools/svgTester/export-graphs.js\
- -i $CONFIGS_DIR\
+ echo "=> Committing new configs and chart ids"
+ cd $SVGS_REPO \
+ && git add --all \
+ && git commit -m "chore: update configs and chart ids" \
+ && cd -
+
+ echo "=> Generating reference SVGs (default views)"
+ rm -rf $REFERENCES_DIR
+ node itsJustJavascript/devTools/svgTester/export-graphs.js \
+ -i $CONFIGS_DIR \
-o $REFERENCES_DIR
- node itsJustJavascript/devTools/svgTester/export-graphs.js\
- -i $CONFIGS_DIR\
- -o $ALL_VIEWS_SVG_DIR\
- -f $CHART_IDS_FILE\
+
+ echo "=> Committing reference SVGs (default views)"
+ cd $SVGS_REPO \
+ && git add --all \
+ && git commit -m 'chore: update reference svgs (default views)' \
+ && cd -
+
+ echo "=> Generating reference SVGs (all views)"
+ rm -rf $ALL_VIEWS_DIR
+ node itsJustJavascript/devTools/svgTester/export-graphs.js \
+ -i $CONFIGS_DIR \
+ -o $ALL_VIEWS_SVG_DIR \
+ -f $CHART_IDS_FILE \
--all-views
+
+ echo "=> Committing reference SVGs (all views)"
+ cd $SVGS_REPO \
+ && git add --all \
+ && git commit -m 'chore: update reference svgs (all views)' \
+ && cd -
}
# show help
diff --git a/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx b/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx
index 36357a0c806..206b71cb57a 100644
--- a/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx
+++ b/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx
@@ -43,11 +43,11 @@ export const getDefaultFailMessage = (manager: ChartManager): string => {
export const getSeriesKey = (
series: LineChartSeries,
- suffix?: string
+ index: number
): string => {
return `${series.seriesName}-${series.color}-${
series.isProjection ? "projection" : ""
- }${suffix ? "-" + suffix : ""}`
+ }-${index}`
}
export const autoDetectSeriesStrategy = (
diff --git a/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss b/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss
index d3f3a464aec..cc0b23bd31a 100644
--- a/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss
+++ b/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss
@@ -26,10 +26,20 @@ $paddingX: 12px; // keep in sync with PADDING_X
}
}
-.ActionButton {
- $light-fill: $gray-10;
- $hover-fill: $gray-20;
- $active-fill: $blue-20;
+div.ActionButton {
+ --light-fill: #{$gray-10};
+ --hover-fill: #{$gray-20};
+ --active-fill: #{$blue-20};
+ --text-color: #{$dark-text};
+
+ &.ActionButton--exploreData {
+ --light-fill: #{$blue-20};
+ --hover-fill: #{$blue-20};
+ --active-fill: #{$blue-10};
+ --text-color: #{$blue-90};
+
+ --hover-decoration: underline;
+ }
height: 100%;
border-radius: 4px;
@@ -43,12 +53,12 @@ $paddingX: 12px; // keep in sync with PADDING_X
height: 100%;
width: 100%;
cursor: pointer;
- color: $dark-text;
+ color: var(--text-color);
font-size: 13px;
font-weight: 500;
padding: 0 $paddingX;
border-radius: inherit;
- background-color: $light-fill;
+ background-color: var(--light-fill);
position: relative;
letter-spacing: 0.01em;
@@ -62,13 +72,14 @@ $paddingX: 12px; // keep in sync with PADDING_X
}
&:hover {
- background-color: $hover-fill;
+ background-color: var(--hover-fill);
+ text-decoration: var(--hover-decoration);
}
&:active,
&.active {
color: $active-text;
- background-color: $active-fill;
+ background-color: var(--active-fill);
}
}
diff --git a/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx b/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx
index dadc693d304..13e0e2f1646 100644
--- a/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx
+++ b/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx
@@ -305,7 +305,7 @@ export class ActionButtons extends React.Component<{
{this.hasExploreTheDataButton && (
+
manager?: GrapherManager
instanceRef?: React.RefObject
}
@@ -506,6 +512,11 @@ export class Grapher
isEmbeddedInAnOwidPage?: boolean = this.props.isEmbeddedInAnOwidPage
isEmbeddedInADataPage?: boolean = this.props.isEmbeddedInADataPage
+ chartViewInfo?: Pick<
+ ChartViewInfo,
+ "parentChartSlug" | "queryParamsForParentChart"
+ > = undefined
+
selection =
this.manager?.selection ??
new SelectionArray(
@@ -3521,10 +3532,27 @@ export class Grapher
return this.props.manager
}
+ @computed get canonicalUrlIfIsChartView(): string | undefined {
+ if (!this.chartViewInfo) return undefined
+
+ const { parentChartSlug, queryParamsForParentChart } =
+ this.chartViewInfo
+
+ const combinedQueryParams = {
+ ...queryParamsForParentChart,
+ ...this.changedParams,
+ }
+
+ return `${this.bakedGrapherURL}/${parentChartSlug}${queryParamsToStr(
+ combinedQueryParams
+ )}`
+ }
+
// Get the full url representing the canonical location of this grapher state
@computed get canonicalUrl(): string | undefined {
return (
this.manager?.canonicalUrl ??
+ this.canonicalUrlIfIsChartView ??
(this.baseUrl ? this.baseUrl + this.queryStr : undefined)
)
}
diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts
index 1601f36f65e..dc7143bc79e 100644
--- a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts
+++ b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts
@@ -5,6 +5,9 @@ import type { GrapherProgrammaticInterface } from "./Grapher"
export const GRAPHER_EMBEDDED_FIGURE_ATTR = "data-grapher-src"
export const GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR = "data-grapher-config"
+export const GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR =
+ "data-grapher-chart-view-config"
+
export const GRAPHER_PAGE_BODY_CLASS = "StandaloneGrapherOrExplorerPage"
export const GRAPHER_IS_IN_IFRAME_CLASS = "IsInIframe"
export const GRAPHER_TIMELINE_CLASS = "timeline-component"
diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts
index 8d52675e86c..13ca26a69cf 100644
--- a/packages/@ourworldindata/grapher/src/index.ts
+++ b/packages/@ourworldindata/grapher/src/index.ts
@@ -10,6 +10,7 @@ export { ChartDimension } from "./chart/ChartDimension"
export {
GRAPHER_EMBEDDED_FIGURE_ATTR,
GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR,
+ GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR,
GRAPHER_PAGE_BODY_CLASS,
GRAPHER_IS_IN_IFRAME_CLASS,
DEFAULT_GRAPHER_WIDTH,
diff --git a/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx b/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx
index dd54175a663..3369bbbfa26 100644
--- a/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx
+++ b/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx
@@ -279,8 +279,8 @@ class Lines extends React.Component {
private renderLines(): React.ReactElement {
return (
<>
- {this.props.series.map((series) => (
-
+ {this.props.series.map((series, index) => (
+
{this.renderLine(series)}
{this.renderLineMarkers(series)}
@@ -556,7 +556,7 @@ export class LineChart
y2={verticalAxis.range[1]}
stroke="rgba(180,180,180,.4)"
/>
- {this.renderSeries.map((series) => {
+ {this.renderSeries.map((series, index) => {
const value = series.points.find(
(point) => point.x === activeX
)
@@ -574,7 +574,7 @@ export class LineChart
return (
!!series.isProjection))
- seriesToShow = seriesToShow.filter((series) => series.isProjection)
+ // If there are any projections, ignore non-projection legends (bit of a hack)
+ let series = this.series
+ if (series.some((series) => !!series.isProjection))
+ series = series.filter((series) => series.isProjection)
+
+ // Deduplicate series by seriesName to avoid showing the same label multiple times
+ const deduplicatedSeries: LineChartSeries[] = []
+ const seriesGroupedByName = groupBy(series, "seriesName")
+ for (const duplicates of Object.values(seriesGroupedByName)) {
+ // keep only the label for the series with the most recent data
+ // (series are sorted by time, so we can just take the last one)
+ deduplicatedSeries.push(last(duplicates)!)
+ }
- return seriesToShow.map((series) => {
+ return deduplicatedSeries.map((series) => {
const { seriesName, color } = series
const lastValue = last(series.points)!.y
return {
diff --git a/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx b/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx
index ed14c1c17d2..dbcc3d0127d 100644
--- a/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx
+++ b/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx
@@ -39,6 +39,7 @@ import {
MARKER_MARGIN,
NON_FOCUSED_TEXT_COLOR,
} from "./LineLegendConstants.js"
+import { getSeriesKey } from "./LineLegendHelpers"
export interface LineLabelSeries extends ChartSeries {
label: string
@@ -150,7 +151,7 @@ class LineLabels extends React.Component<{
@computed private get textLabels(): React.ReactElement {
return (
- {this.markers.map(({ series, labelText }) => {
+ {this.markers.map(({ series, labelText }, index) => {
const textColor =
!series.focus?.background || series.hover?.active
? darkenColorForText(series.color)
@@ -164,7 +165,7 @@ class LineLabels extends React.Component<{
return series.textWrap instanceof TextWrap ? (
@@ -197,12 +198,12 @@ class LineLabels extends React.Component<{
if (!markersWithAnnotations) return
return (
- {markersWithAnnotations.map(({ series, labelText }) => {
+ {markersWithAnnotations.map(({ series, labelText }, index) => {
if (!series.annotationTextWrap) return
return (
@@ -232,7 +233,7 @@ class LineLabels extends React.Component<{
if (!this.props.needsConnectorLines) return
return (
- {this.markers.map(({ series, connectorLine }) => {
+ {this.markers.map(({ series, connectorLine }, index) => {
const { x1, x2 } = connectorLine
const {
level,
@@ -253,7 +254,7 @@ class LineLabels extends React.Component<{
return (
- {this.props.series.map((series) => {
+ {this.props.series.map((series, index) => {
const x =
this.anchor === "start"
? series.origBounds.x
: series.origBounds.x - series.bounds.width
return (
this.props.onMouseOver?.(series)}
onMouseLeave={() =>
this.props.onMouseLeave?.(series)
diff --git a/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts b/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts
index 17310a4d4e6..286c5cf7f36 100644
--- a/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts
+++ b/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts
@@ -245,3 +245,7 @@ export function computeCandidateScores(
return scoreMap
}
+
+export function getSeriesKey(series: PlacedSeries, index: number): string {
+ return `${series.seriesName}-${index}`
+}
diff --git a/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx b/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx
index e8c9930dab4..b89031653bc 100644
--- a/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx
+++ b/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx
@@ -875,9 +875,7 @@ export const DownloadModalDataTab = (props: DownloadModalProps) => {
onClick={() => onDownloadClick(CsvDownloadType.Full)}
tracking={
"chart_download_full_data--" +
- serverSideDownloadAvailable
- ? "server"
- : "client"
+ (serverSideDownloadAvailable ? "server" : "client")
}
/>
{
}
tracking={
"chart_download_filtered_data--" +
- serverSideDownloadAvailable
- ? "server"
- : "client"
+ (serverSideDownloadAvailable ? "server" : "client")
}
/>
diff --git a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts
index cbb50d10c51..06c200041d1 100644
--- a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts
+++ b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts
@@ -86,6 +86,31 @@ export type EnrichedBlockChart = {
tabs?: ChartTabKeyword[]
} & EnrichedBlockWithParseErrors
+export type RawBlockNarrativeChartValue = {
+ name?: string
+ height?: string
+ row?: string
+ column?: string
+ // TODO: position is used as a classname apparently? Should be renamed or split
+ position?: string
+ caption?: string
+}
+
+export type RawBlockNarrativeChart = {
+ type: "narrative-chart"
+ value: RawBlockNarrativeChartValue | string
+}
+
+export type EnrichedBlockNarrativeChart = {
+ type: "narrative-chart"
+ name: string
+ height?: string
+ row?: string
+ column?: string
+ position?: ChartPositionChoice
+ caption?: Span[]
+} & EnrichedBlockWithParseErrors
+
export type RawBlockCode = {
type: "code"
value: RawBlockText[]
@@ -950,6 +975,7 @@ export type OwidRawGdocBlock =
| RawBlockAside
| RawBlockCallout
| RawBlockChart
+ | RawBlockNarrativeChart
| RawBlockCode
| RawBlockDonorList
| RawBlockScroller
@@ -1001,6 +1027,7 @@ export type OwidEnrichedGdocBlock =
| EnrichedBlockAside
| EnrichedBlockCallout
| EnrichedBlockChart
+ | EnrichedBlockNarrativeChart
| EnrichedBlockCode
| EnrichedBlockDonorList
| EnrichedBlockScroller
diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts
index a6f35022ea3..318c5d1e5b2 100644
--- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts
+++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts
@@ -13,6 +13,7 @@ import {
} from "./ArchieMlComponents.js"
import { MinimalTag } from "../dbTypes/Tags.js"
import { DbEnrichedLatestWork } from "../domainTypes/Author.js"
+import { QueryParams } from "../domainTypes/Various.js"
export enum OwidGdocPublicationContext {
unlisted = "unlisted",
@@ -53,6 +54,15 @@ export interface LinkedChart {
indicatorId?: number // in case of a datapage
}
+// An object containing metadata needed for embedded narrative charts
+export interface ChartViewInfo {
+ name: string
+ title: string
+ chartConfigId: string
+ parentChartSlug: string
+ queryParamsForParentChart: QueryParams
+}
+
/**
* A linked indicator is derived from a linked grapher's config (see: getVariableOfDatapageIfApplicable)
* e.g. https://ourworldindata.org/grapher/tomato-production -> config for grapher with { slug: "tomato-production" } -> indicator metadata
@@ -271,6 +281,7 @@ export enum OwidGdocLinkType {
Url = "url",
Grapher = "grapher",
Explorer = "explorer",
+ ChartView = "chart-view",
}
export interface OwidGdocLinkJSON {
diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts
index f5fd8971661..a8a3f80db50 100644
--- a/packages/@ourworldindata/types/src/index.ts
+++ b/packages/@ourworldindata/types/src/index.ts
@@ -287,6 +287,8 @@ export {
SocialLinkType,
type RawSocialLink,
type EnrichedSocialLink,
+ type RawBlockNarrativeChart,
+ type EnrichedBlockNarrativeChart,
} from "./gdocTypes/ArchieMlComponents.js"
export {
ChartConfigType,
@@ -330,6 +332,7 @@ export {
type OwidGdocContent,
type OwidGdocIndexItem,
extractGdocIndexItem,
+ type ChartViewInfo,
} from "./gdocTypes/Gdoc.js"
export {
diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts
index cd2978a218b..547e9404216 100644
--- a/packages/@ourworldindata/utils/src/Util.ts
+++ b/packages/@ourworldindata/utils/src/Util.ts
@@ -1711,6 +1711,7 @@ export function traverseEnrichedBlock(
type: P.union(
"chart-story",
"chart",
+ "narrative-chart",
"code",
"donors",
"horizontal-rule",
diff --git a/packages/@ourworldindata/utils/src/metadataHelpers.ts b/packages/@ourworldindata/utils/src/metadataHelpers.ts
index 6a99a3d3d2b..9fcf200dd47 100644
--- a/packages/@ourworldindata/utils/src/metadataHelpers.ts
+++ b/packages/@ourworldindata/utils/src/metadataHelpers.ts
@@ -73,16 +73,11 @@ export const getETLPathComponents = (path: string): ETLPathComponents => {
export const formatAuthors = ({
authors,
- requireMax,
forBibtex,
}: {
authors: string[]
- requireMax?: boolean
forBibtex?: boolean
}): string => {
- if (requireMax && !authors.includes("Max Roser"))
- authors = [...authors, "Max Roser"]
-
let authorsText = authors.slice(0, -1).join(forBibtex ? " and " : ", ")
if (authorsText.length === 0) authorsText = authors[0]
else authorsText += ` and ${last(authors)}`
diff --git a/public/owid-logo.svg b/public/owid-logo.svg
index 38643f2e383..5c7ea553986 100644
--- a/public/owid-logo.svg
+++ b/public/owid-logo.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/site/Byline.tsx b/site/Byline.tsx
index 0042eaff219..738f35769f5 100644
--- a/site/Byline.tsx
+++ b/site/Byline.tsx
@@ -2,11 +2,9 @@ import { formatAuthors } from "./clientFormatting.js"
export const Byline = ({
authors,
- withMax,
override,
}: {
authors: string[]
- withMax: boolean
override?: string
}) => {
return (
@@ -20,7 +18,6 @@ export const Byline = ({
) : (
{`by ${formatAuthors({
authors,
- requireMax: withMax,
})}`}
)}
),
@@ -135,7 +135,7 @@ export function ChartViewIndexPage() {
}, [admin])
return (
- 
+