From 124ccea1b3b5b75ce90a17f89d5d4eb95da9a31a Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Thu, 19 Oct 2023 18:11:28 -0400 Subject: [PATCH 01/20] =?UTF-8?q?=F0=9F=8E=89=20gdocs=20tables=20prelimina?= =?UTF-8?q?ry=20types=20and=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/model/Gdoc/Gdoc.ts | 1 + db/model/Gdoc/enrichedToRaw.ts | 20 ++ db/model/Gdoc/exampleEnrichedBlocks.ts | 87 ++++++++ db/model/Gdoc/gdocToArchie.ts | 188 +++++++++++------- db/model/Gdoc/rawToArchie.ts | 45 ++++- db/model/Gdoc/rawToEnriched.ts | 103 ++++++++++ packages/@ourworldindata/utils/src/Util.ts | 9 + packages/@ourworldindata/utils/src/index.ts | 7 + .../@ourworldindata/utils/src/owidTypes.ts | 40 ++++ site/gdocs/ArticleBlock.tsx | 5 + 10 files changed, 434 insertions(+), 71 deletions(-) diff --git a/db/model/Gdoc/Gdoc.ts b/db/model/Gdoc/Gdoc.ts index 2f112480df5..2876658e182 100644 --- a/db/model/Gdoc/Gdoc.ts +++ b/db/model/Gdoc/Gdoc.ts @@ -633,6 +633,7 @@ export class Gdoc extends BaseEntity implements OwidGdocInterface { "simple-text", "sticky-left", "sticky-right", + "table", "text" ), }, diff --git a/db/model/Gdoc/enrichedToRaw.ts b/db/model/Gdoc/enrichedToRaw.ts index 0ea822ada83..d4757fdb8ad 100644 --- a/db/model/Gdoc/enrichedToRaw.ts +++ b/db/model/Gdoc/enrichedToRaw.ts @@ -34,6 +34,7 @@ import { RawBlockAlign, RawBlockEntrySummary, RawBlockVideo, + RawBlockTable, } from "@ourworldindata/utils" import { spanToHtmlString } from "./gdocUtils.js" import { match, P } from "ts-pattern" @@ -400,5 +401,24 @@ export function enrichedBlockToRawBlock( }, } }) + .with({ type: "table" }, (b): RawBlockTable => { + return { + type: b.type, + value: { + template: b.template, + rows: b.rows.map((row) => ({ + type: row.type, + value: { + cells: row.cells.map((cell) => ({ + type: cell.type, + value: cell.content.map( + enrichedBlockToRawBlock + ), + })), + }, + })), + }, + } + }) .exhaustive() } diff --git a/db/model/Gdoc/exampleEnrichedBlocks.ts b/db/model/Gdoc/exampleEnrichedBlocks.ts index e63c8169da4..88486d475e9 100644 --- a/db/model/Gdoc/exampleEnrichedBlocks.ts +++ b/db/model/Gdoc/exampleEnrichedBlocks.ts @@ -441,4 +441,91 @@ export const enrichedBlockExamples: Record< items: [{ text: "Hello", slug: "#link-to-something" }], parseErrors: [], }, + table: { + type: "table", + template: "header-row", + rows: [ + { + type: "table-row", + cells: [ + { + type: "table-cell", + content: [ + { + type: "text", + value: [ + { + spanType: "span-simple-text", + text: "City", + }, + ], + parseErrors: [], + }, + { + type: "text", + value: [ + { + spanType: "span-simple-text", + text: "Continent", + }, + ], + parseErrors: [], + }, + ], + }, + { + type: "table-cell", + content: [ + { + type: "text", + value: [ + { + spanType: "span-simple-text", + text: "Wellington", + }, + ], + parseErrors: [], + }, + { + type: "text", + value: [ + { + spanType: "span-simple-text", + text: "Zealandia", + }, + ], + parseErrors: [], + }, + ], + }, + { + type: "table-cell", + content: [ + { + type: "text", + value: [ + { + spanType: "span-simple-text", + text: "Addis Ababa", + }, + ], + parseErrors: [], + }, + { + type: "text", + value: [ + { + spanType: "span-simple-text", + text: "Africa", + }, + ], + parseErrors: [], + }, + ], + }, + ], + }, + ], + parseErrors: [], + }, } diff --git a/db/model/Gdoc/gdocToArchie.ts b/db/model/Gdoc/gdocToArchie.ts index a84c562c683..fb0e9444a68 100644 --- a/db/model/Gdoc/gdocToArchie.ts +++ b/db/model/Gdoc/gdocToArchie.ts @@ -6,95 +6,145 @@ import { RawBlockHorizontalRule, RawBlockHeading, isNil, + RawBlockTable, + RawBlockTableRow, + OwidRawGdocBlock, + RawBlockTableCell, + RawBlockText, } from "@ourworldindata/utils" import { spanToHtmlString } from "./gdocUtils.js" import { OwidRawGdocBlockToArchieMLString } from "./rawToArchie.js" import { match, P } from "ts-pattern" +function paragraphToString(paragraph: docs_v1.Schema$Paragraph): string { + let text = "" + let isInList = false + + // this is a list + const needsBullet = !isNil(paragraph.bullet) + if (needsBullet && !isInList) { + isInList = true + text += `\n[.list]\n` + } else if (!needsBullet && isInList) { + isInList = false + text += `[]\n` + } + + if (paragraph.elements) { + // all values in the element + const values: docs_v1.Schema$ParagraphElement[] = paragraph.elements + + let idx = 0 + + const taggedText = function (text: string): string { + if (paragraph.paragraphStyle?.namedStyleType?.includes("HEADING")) { + const headingLevel = + paragraph.paragraphStyle.namedStyleType.replace( + "HEADING_", + "" + ) + + const heading: RawBlockHeading = { + type: "heading", + value: { + text: text.trim(), + level: headingLevel, + }, + } + return `\n${OwidRawGdocBlockToArchieMLString(heading)}` + } + return text + } + let elementText = "" + for (const value of values) { + // we only need to add a bullet to the first value, so we check + const isFirstValue = idx === 0 + + // prepend an asterisk if this is a list item + const prefix = needsBullet && isFirstValue ? "* " : "" + + // concat the text + const parsedParagraph = parseParagraph(value) + const fragmentText = match(parsedParagraph) + .with( + { type: P.union("horizontal-rule") }, + OwidRawGdocBlockToArchieMLString + ) + .with({ spanType: P.any }, (s) => spanToHtmlString(s)) + .with(P.nullish, () => "") + .exhaustive() + elementText += `${prefix}${fragmentText}` + idx++ + } + text += taggedText(elementText) + } + return text +} + +function tableToString( + table: docs_v1.Schema$StructuralElement["table"] +): string { + if (!table) return "" + let text = "" + const { tableRows = [] } = table + + const rows: RawBlockTableRow[] = [] + + for (const tableRow of tableRows) { + const rawRow: RawBlockTableRow = { + type: "table-row", + value: { + cells: [], + }, + } + const { tableCells = [] } = tableRow + for (const tableCell of tableCells) { + const rawCell: RawBlockTableCell = { + type: "table-cell", + value: [], + } + const { content = [] } = tableCell + for (const item of content) { + if (item.paragraph) { + const text = paragraphToString(item.paragraph) + const rawTextBlock: RawBlockText = { + type: "text", + value: text, + } + rawCell.value!.push(rawTextBlock) + } + } + rawRow.value!.cells!.push(rawCell) + } + rows.push(rawRow) + } + text += "\n[.+rows]" + for (const row of rows) { + text += `\n${OwidRawGdocBlockToArchieMLString(row)}` + } + text += "\n[]" + return text +} + export async function gdocToArchie( document: docs_v1.Schema$Document ): Promise<{ text: string }> { // prepare the text holder let text = "" - let isInList = false // check if the body key and content key exists, and give up if not if (!document.body) return { text } if (!document.body.content) return { text } // loop through each content element in the body - for (const element of document.body.content) { if (element.paragraph) { - // get the paragraph within the element - const paragraph: docs_v1.Schema$Paragraph = element.paragraph - - // this is a list - const needsBullet = !isNil(paragraph.bullet) - if (needsBullet && !isInList) { - isInList = true - text += `\n[.list]\n` - } else if (!needsBullet && isInList) { - isInList = false - text += `[]\n` - } - - if (paragraph.elements) { - // all values in the element - const values: docs_v1.Schema$ParagraphElement[] = - paragraph.elements - - let idx = 0 - - const taggedText = function (text: string): string { - if ( - paragraph.paragraphStyle?.namedStyleType?.includes( - "HEADING" - ) - ) { - const headingLevel = - paragraph.paragraphStyle.namedStyleType.replace( - "HEADING_", - "" - ) - - const heading: RawBlockHeading = { - type: "heading", - value: { - text: text.trim(), - level: headingLevel, - }, - } - return `\n${OwidRawGdocBlockToArchieMLString(heading)}` - } - return text - } - let elementText = "" - for (const value of values) { - // we only need to add a bullet to the first value, so we check - const isFirstValue = idx === 0 - - // prepend an asterisk if this is a list item - const prefix = needsBullet && isFirstValue ? "* " : "" - - // concat the text - const parsedParagraph = parseParagraph(value) - const fragmentText = match(parsedParagraph) - .with( - { type: P.union("horizontal-rule") }, - OwidRawGdocBlockToArchieMLString - ) - .with({ spanType: P.any }, (s) => spanToHtmlString(s)) - .with(P.nullish, () => "") - .exhaustive() - elementText += `${prefix}${fragmentText}` - idx++ - } - text += taggedText(elementText) - } + text += paragraphToString(element.paragraph) + } else if (element.table) { + text += tableToString(element.table) } } - + console.log("text", text) return { text } } diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts index 59d901d76f4..8f12ad7f416 100644 --- a/db/model/Gdoc/rawToArchie.ts +++ b/db/model/Gdoc/rawToArchie.ts @@ -33,6 +33,8 @@ import { RawBlockAlign, RawBlockEntrySummary, isArray, + RawBlockTable, + RawBlockTableRow, } from "@ourworldindata/utils" import { match } from "ts-pattern" @@ -563,8 +565,45 @@ function* rawBlockEntrySummaryToArchieMLString( yield "{}" } +function* rawBlockRowToArchieMLString( + row: RawBlockTableRow +): Generator { + yield "{.row}" + const cells = row.value.cells + if (cells) { + yield "[.+cells]" + for (const cell of cells) { + const content = cell.value + yield "[.+cell]" + if (content) { + for (const rawBlock of content) + yield* OwidRawGdocBlockToArchieMLStringGenerator(rawBlock) + } + yield "[]" + } + yield "[]" + } + yield "{}" +} + +function* rawBlockTableToArchieMLString( + block: RawBlockTable +): Generator { + yield "{.table}" + yield* propertyToArchieMLString("template", block.value) + const rows = block?.value?.rows + if (rows) { + yield "[.+rows]" + for (const row of rows) { + yield* rawBlockRowToArchieMLString(row) + } + yield "[]" + } + yield "{}" +} + export function* OwidRawGdocBlockToArchieMLStringGenerator( - block: OwidRawGdocBlock + block: OwidRawGdocBlock | RawBlockTableRow ): Generator { const content = match(block) .with( @@ -624,12 +663,14 @@ export function* OwidRawGdocBlockToArchieMLStringGenerator( ) .with({ type: "align" }, rawBlockAlignToArchieMLString) .with({ type: "entry-summary" }, rawBlockEntrySummaryToArchieMLString) + .with({ type: "table" }, rawBlockTableToArchieMLString) + .with({ type: "table-row" }, rawBlockRowToArchieMLString) .exhaustive() yield* content } export function OwidRawGdocBlockToArchieMLString( - block: OwidRawGdocBlock + block: OwidRawGdocBlock | RawBlockTableRow ): string { const lines = [...OwidRawGdocBlockToArchieMLStringGenerator(block)] return [...lines, ""].join("\n") diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts index 35ade73c76e..cd785161346 100644 --- a/db/model/Gdoc/rawToEnriched.ts +++ b/db/model/Gdoc/rawToEnriched.ts @@ -99,6 +99,11 @@ import { RawBlockEntrySummary, EnrichedBlockEntrySummary, EnrichedBlockEntrySummaryItem, + RawBlockTable, + EnrichedBlockTable, + EnrichedBlockTableRow, + TableTemplate, + EnrichedBlockTableCell, } from "@ourworldindata/utils" import { checkIsInternalLink } from "@ourworldindata/components" import { @@ -180,6 +185,7 @@ export function parseRawBlocksToEnrichedBlocks( .with({ type: "expandable-paragraph" }, parseExpandableParagraph) .with({ type: "align" }, parseAlign) .with({ type: "entry-summary" }, parseEntrySummary) + .with({ type: "table" }, parseTable) .exhaustive() } @@ -790,6 +796,103 @@ const parseRecirc = (raw: RawBlockRecirc): EnrichedBlockRecirc => { } } +export const parseTable = (raw: RawBlockTable): EnrichedBlockTable => { + const createError = ( + error: ParseError, + template: TableTemplate = "header-row", + rows: EnrichedBlockTableRow[] = [] + ): EnrichedBlockTable => ({ + type: "table", + template, + rows, + parseErrors: [error], + }) + + const parseErrors: ParseError[] = [] + + const validTemplates: TableTemplate[] = [ + "header-row", + "header-column", + "header-column-row", + ] + function validateTableTemplate( + template: unknown + ): template is TableTemplate { + return validTemplates.includes(template as TableTemplate) + } + + const template = raw.value?.template + if (!validateTableTemplate(template)) + return createError({ + message: `Invalid table template "${template}". Must be one of ${validTemplates.join( + ", " + )}`, + }) + + const rows = raw.value?.rows + const enrichedRows: EnrichedBlockTableRow[] = [] + if (!rows) + return createError({ + message: "Table must have at least one row", + }) + + for (const [rowIndex, row] of rows.entries()) { + const enrichedCells: EnrichedBlockTableCell[] = [] + const cells = row.value.cells + if (!cells) { + parseErrors.push({ + message: `Row ${rowIndex} is missing cells`, + }) + } else { + for (const [cellIndex, cell] of cells.entries()) { + const enrichedCellContent: OwidEnrichedGdocBlock[] = [] + const content = cell.value + if (!content || !content.length) { + parseErrors.push({ + message: `Cell (${rowIndex}, ${cellIndex}) is missing content`, + }) + } else { + for (const [ + contentIndex, + contentItem, + ] of content.entries()) { + const enrichedContent = + parseRawBlocksToEnrichedBlocks(contentItem) + if (!enrichedContent) { + parseErrors.push({ + message: `Cell (${rowIndex}, ${cellIndex}) content item ${contentIndex} is invalid`, + }) + } else if (enrichedContent.parseErrors.length) { + parseErrors.push( + ...enrichedContent.parseErrors.map((error) => ({ + message: `Cell (${rowIndex}, ${cellIndex}) content item ${contentIndex} has error: ${error.message}`, + })) + ) + } else { + enrichedCellContent.push(enrichedContent) + } + } + enrichedCells.push({ + type: "table-cell", + content: enrichedCellContent, + }) + } + } + enrichedRows.push({ + type: "table-row", + cells: enrichedCells, + }) + } + } + + return { + type: "table", + rows: enrichedRows, + template, + parseErrors, + } +} + export const parseText = (raw: RawBlockText): EnrichedBlockText => { const createError = ( error: ParseError, diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index a6e596534a4..da0a6cc4a37 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -1602,6 +1602,15 @@ export function traverseEnrichedBlocks( traverseEnrichedBlocks(node, callback, spanCallback) }) }) + .with({ type: "table" }, (table) => { + table.rows.forEach((row) => { + row.cells.forEach((cell) => { + cell.content.forEach((node) => { + traverseEnrichedBlocks(node, callback, spanCallback) + }) + }) + }) + }) .with( { type: P.union( diff --git a/packages/@ourworldindata/utils/src/index.ts b/packages/@ourworldindata/utils/src/index.ts index 97a91a7cbf5..0b86c54e8f8 100644 --- a/packages/@ourworldindata/utils/src/index.ts +++ b/packages/@ourworldindata/utils/src/index.ts @@ -78,6 +78,9 @@ export { type EnrichedBlockEntrySummaryItem, type EntryMeta, type EntryNode, + type EnrichedBlockTable, + type EnrichedBlockTableRow, + type EnrichedBlockTableCell, EPOCH_DATE, type FilterFnPostRestApi, type FormattedPost, @@ -153,6 +156,10 @@ export { type RawBlockText, type RawBlockTopicPageIntro, type RawBlockUrl, + type TableTemplate, + type RawBlockTable, + type RawBlockTableRow, + type RawBlockTableCell, type RawChartStoryValue, type RawRecircLink, type RawDetail, diff --git a/packages/@ourworldindata/utils/src/owidTypes.ts b/packages/@ourworldindata/utils/src/owidTypes.ts index a2a88d4d904..0d3802d4172 100644 --- a/packages/@ourworldindata/utils/src/owidTypes.ts +++ b/packages/@ourworldindata/utils/src/owidTypes.ts @@ -1138,6 +1138,44 @@ export type EnrichedBlockEntrySummary = { items: EnrichedBlockEntrySummaryItem[] } & EnrichedBlockWithParseErrors +export type TableTemplate = "header-column" | "header-row" | "header-column-row" + +export type RawBlockTable = { + type: "table" + value?: { + template?: TableTemplate + rows?: RawBlockTableRow[] + } +} + +export interface RawBlockTableRow { + type: "table-row" + value: { + cells?: RawBlockTableCell[] + } +} + +export interface RawBlockTableCell { + type: "table-cell" + value?: OwidRawGdocBlock[] +} + +export type EnrichedBlockTable = { + type: "table" + template: TableTemplate + rows: EnrichedBlockTableRow[] +} & EnrichedBlockWithParseErrors + +export interface EnrichedBlockTableRow { + type: "table-row" + cells: EnrichedBlockTableCell[] +} + +export interface EnrichedBlockTableCell { + type: "table-cell" + content: OwidEnrichedGdocBlock[] +} + export type Ref = { id: string // Can be -1 @@ -1184,6 +1222,7 @@ export type OwidRawGdocBlock = | RawBlockKeyInsights | RawBlockAlign | RawBlockEntrySummary + | RawBlockTable export type OwidEnrichedGdocBlock = | EnrichedBlockAllCharts @@ -1219,6 +1258,7 @@ export type OwidEnrichedGdocBlock = | EnrichedBlockResearchAndWriting | EnrichedBlockAlign | EnrichedBlockEntrySummary + | EnrichedBlockTable export enum OwidGdocPublicationContext { unlisted = "unlisted", diff --git a/site/gdocs/ArticleBlock.tsx b/site/gdocs/ArticleBlock.tsx index ba6e275d21d..e21da6833d3 100644 --- a/site/gdocs/ArticleBlock.tsx +++ b/site/gdocs/ArticleBlock.tsx @@ -575,6 +575,11 @@ export default function ArticleBlock({ ))} )) + .with({ type: "table" }, (block) => ( +
+ I'm a {block.type} +
+ )) .exhaustive() return ( From 873abfc21ff3d27ae7c2a9a32f9bfbe3c1d6cb20 Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Mon, 23 Oct 2023 13:54:34 +0000 Subject: [PATCH 02/20] =?UTF-8?q?=F0=9F=8E=89=20gdocs=20table=20rendering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/model/Gdoc/gdocToArchie.ts | 1 - db/model/Gdoc/rawToEnriched.ts | 5 +++-- site/gdocs/ArticleBlock.tsx | 12 +++++++++--- site/gdocs/centered-article.scss | 27 +++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/db/model/Gdoc/gdocToArchie.ts b/db/model/Gdoc/gdocToArchie.ts index fb0e9444a68..f834c0eea32 100644 --- a/db/model/Gdoc/gdocToArchie.ts +++ b/db/model/Gdoc/gdocToArchie.ts @@ -144,7 +144,6 @@ export async function gdocToArchie( text += tableToString(element.table) } } - console.log("text", text) return { text } } diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts index cd785161346..7d0971ebe6f 100644 --- a/db/model/Gdoc/rawToEnriched.ts +++ b/db/model/Gdoc/rawToEnriched.ts @@ -848,8 +848,9 @@ export const parseTable = (raw: RawBlockTable): EnrichedBlockTable => { const enrichedCellContent: OwidEnrichedGdocBlock[] = [] const content = cell.value if (!content || !content.length) { - parseErrors.push({ - message: `Cell (${rowIndex}, ${cellIndex}) is missing content`, + enrichedCells.push({ + type: "table-cell", + content: [], }) } else { for (const [ diff --git a/site/gdocs/ArticleBlock.tsx b/site/gdocs/ArticleBlock.tsx index e21da6833d3..ce4fecca18a 100644 --- a/site/gdocs/ArticleBlock.tsx +++ b/site/gdocs/ArticleBlock.tsx @@ -33,6 +33,7 @@ import { KeyInsights } from "./KeyInsights.js" import { ResearchAndWriting } from "./ResearchAndWriting.js" import { AllCharts } from "./AllCharts.js" import Video from "./Video.js" +import { Table } from "./Table.js" export type Container = | "default" @@ -84,6 +85,7 @@ const layouts: { [key in Container]: Layouts} = { ["sticky-right-left-column"]: "grid span-cols-5 grid grid-cols-5 span-md-cols-10 grid-md-cols-10 col-md-start-2 span-sm-cols-12 grid-sm-cols-12 col-sm-start-1", ["sticky-right-right-column"]: "span-cols-7 grid-cols-7 span-md-cols-10 grid-md-cols-10 col-md-start-2 span-sm-cols-12 grid-sm-cols-12 col-sm-start-1", ["sticky-right"]: "grid span-cols-12 col-start-2", + ["table"]: "col-start-4 span-cols-8 col-md-start-2 span-md-cols-12", ["text"]: "col-start-5 span-cols-6 col-md-start-3 span-md-cols-10 span-sm-cols-12 col-sm-start-2", ["toc"]: "grid grid-cols-8 col-start-4 span-cols-8 grid-md-cols-10 col-md-start-3 span-md-cols-10 grid-sm-cols-12 span-sm-cols-12 col-sm-start-2", ["topic-page-intro"]: "grid col-start-2 span-cols-12", @@ -576,9 +578,13 @@ export default function ArticleBlock({ )) .with({ type: "table" }, (block) => ( -
- I'm a {block.type} -
+ )) .exhaustive() diff --git a/site/gdocs/centered-article.scss b/site/gdocs/centered-article.scss index 584d0d51b86..3f55ef3e411 100644 --- a/site/gdocs/centered-article.scss +++ b/site/gdocs/centered-article.scss @@ -458,6 +458,33 @@ h3.article-block__heading.has-supertitle { } } +.article-block__table { + border-collapse: collapse; + margin-bottom: 32px; + + th[scope="col"], + th[scope="row"] { + background-color: $blue-10; + p { + font-weight: 700; + margin-bottom: 0; + } + } + + .table-cell { + border: 1px solid $blue-20; + padding: 8px; + } + + .article-block__text, + .article-block__list { + @include body-3-medium; + &:last-child { + margin-bottom: 0; + } + } +} + .article-block__pull-quote { @include h1-bold-italic; border-bottom: 1px solid #dbe5f0; From bf37712259ba58b462681889b02a1fb5a9c8ed84 Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Mon, 23 Oct 2023 14:37:07 +0000 Subject: [PATCH 03/20] =?UTF-8?q?=E2=9C=85=20fix=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/model/Gdoc/Gdoc.ts | 1 + db/model/Gdoc/gdocToArchie.ts | 2 -- db/model/Gdoc/rawToArchie.ts | 4 ++-- site/gdocs/ArticleBlock.tsx | 2 +- site/gdocs/centered-article.scss | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/db/model/Gdoc/Gdoc.ts b/db/model/Gdoc/Gdoc.ts index 2876658e182..938e71f5930 100644 --- a/db/model/Gdoc/Gdoc.ts +++ b/db/model/Gdoc/Gdoc.ts @@ -182,6 +182,7 @@ export class Gdoc extends BaseEntity implements OwidGdocInterface { // Convert the doc to ArchieML syntax const { text } = await gdocToArchie(data) + console.log("text", text) // Convert the ArchieML to our enriched JSON structure this.content = archieToEnriched(text) diff --git a/db/model/Gdoc/gdocToArchie.ts b/db/model/Gdoc/gdocToArchie.ts index f834c0eea32..3d84c96cf2a 100644 --- a/db/model/Gdoc/gdocToArchie.ts +++ b/db/model/Gdoc/gdocToArchie.ts @@ -6,9 +6,7 @@ import { RawBlockHorizontalRule, RawBlockHeading, isNil, - RawBlockTable, RawBlockTableRow, - OwidRawGdocBlock, RawBlockTableCell, RawBlockText, } from "@ourworldindata/utils" diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts index 8f12ad7f416..5706a060d0d 100644 --- a/db/model/Gdoc/rawToArchie.ts +++ b/db/model/Gdoc/rawToArchie.ts @@ -568,13 +568,13 @@ function* rawBlockEntrySummaryToArchieMLString( function* rawBlockRowToArchieMLString( row: RawBlockTableRow ): Generator { - yield "{.row}" + yield "{.table-row}" const cells = row.value.cells if (cells) { yield "[.+cells]" for (const cell of cells) { const content = cell.value - yield "[.+cell]" + yield "[.+table-cell]" if (content) { for (const rawBlock of content) yield* OwidRawGdocBlockToArchieMLStringGenerator(rawBlock) diff --git a/site/gdocs/ArticleBlock.tsx b/site/gdocs/ArticleBlock.tsx index ce4fecca18a..775d498c1db 100644 --- a/site/gdocs/ArticleBlock.tsx +++ b/site/gdocs/ArticleBlock.tsx @@ -85,7 +85,7 @@ const layouts: { [key in Container]: Layouts} = { ["sticky-right-left-column"]: "grid span-cols-5 grid grid-cols-5 span-md-cols-10 grid-md-cols-10 col-md-start-2 span-sm-cols-12 grid-sm-cols-12 col-sm-start-1", ["sticky-right-right-column"]: "span-cols-7 grid-cols-7 span-md-cols-10 grid-md-cols-10 col-md-start-2 span-sm-cols-12 grid-sm-cols-12 col-sm-start-1", ["sticky-right"]: "grid span-cols-12 col-start-2", - ["table"]: "col-start-4 span-cols-8 col-md-start-2 span-md-cols-12", + ["table"]: "col-start-5 span-cols-6 col-md-start-3 span-md-cols-10 span-sm-cols-12 col-sm-start-2", ["text"]: "col-start-5 span-cols-6 col-md-start-3 span-md-cols-10 span-sm-cols-12 col-sm-start-2", ["toc"]: "grid grid-cols-8 col-start-4 span-cols-8 grid-md-cols-10 col-md-start-3 span-md-cols-10 grid-sm-cols-12 span-sm-cols-12 col-sm-start-2", ["topic-page-intro"]: "grid col-start-2 span-cols-12", diff --git a/site/gdocs/centered-article.scss b/site/gdocs/centered-article.scss index 3f55ef3e411..89bbd028c53 100644 --- a/site/gdocs/centered-article.scss +++ b/site/gdocs/centered-article.scss @@ -473,7 +473,7 @@ h3.article-block__heading.has-supertitle { .table-cell { border: 1px solid $blue-20; - padding: 8px; + padding: 16px; } .article-block__text, From 8bcc792bc29305884e4df61170e3d21d181a37bb Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Mon, 23 Oct 2023 15:24:26 +0000 Subject: [PATCH 04/20] =?UTF-8?q?=F0=9F=8E=89=20restore=20Table.tsx=20comp?= =?UTF-8?q?onent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- site/gdocs/Table.tsx | 55 ++++++++++++++++++++++++++++++++ site/gdocs/centered-article.scss | 12 +++++++ 2 files changed, 67 insertions(+) create mode 100644 site/gdocs/Table.tsx diff --git a/site/gdocs/Table.tsx b/site/gdocs/Table.tsx new file mode 100644 index 00000000000..db66c440fa1 --- /dev/null +++ b/site/gdocs/Table.tsx @@ -0,0 +1,55 @@ +import React from "react" +import { EnrichedBlockTable } from "@ourworldindata/utils" +import ArticleBlock from "./ArticleBlock.js" + +export type TableProps = { + className?: string +} & EnrichedBlockTable + +export function Table(props: TableProps) { + const { className, rows, template } = props + const isFirstColumnHeader = + template === "header-column-row" || template === "header-column" + const isFirstRowHeader = + template === "header-column-row" || template === "header-row" + + return ( +
+ {rows.map((row, i) => ( + + {row.cells.map((cell, j) => { + if (isFirstColumnHeader && j === 0) { + return ( + + ) + } + if (isFirstRowHeader && i === 0) { + return ( + + ) + } + return ( + + ) + })} + + ))} +
+ {cell.content.map((block, k) => { + return ( + + ) + })} + + {cell.content.map((block, k) => { + return ( + + ) + })} + + {cell.content.map((block, k) => { + return + })} +
+ ) +} diff --git a/site/gdocs/centered-article.scss b/site/gdocs/centered-article.scss index 89bbd028c53..8d854d9be88 100644 --- a/site/gdocs/centered-article.scss +++ b/site/gdocs/centered-article.scss @@ -462,6 +462,18 @@ h3.article-block__heading.has-supertitle { border-collapse: collapse; margin-bottom: 32px; + &.article-block__table--header-column { + th { + text-align: left; + } + } + + &.article-block__table--header-column-row { + th:not([scope="row"]):not([scope="col"]) { + text-align: left; + } + } + th[scope="col"], th[scope="row"] { background-color: $blue-10; From 02595788201ae4d75c3a60a332c93bb88dd27730 Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Mon, 23 Oct 2023 21:58:14 +0000 Subject: [PATCH 05/20] =?UTF-8?q?=F0=9F=90=9B=20fix=20list=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/model/Gdoc/gdocToArchie.ts | 21 ++++++++++++--------- site/gdocs/centered-article.scss | 13 ++++--------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/db/model/Gdoc/gdocToArchie.ts b/db/model/Gdoc/gdocToArchie.ts index 3d84c96cf2a..8d41c82a39e 100644 --- a/db/model/Gdoc/gdocToArchie.ts +++ b/db/model/Gdoc/gdocToArchie.ts @@ -14,17 +14,19 @@ import { spanToHtmlString } from "./gdocUtils.js" import { OwidRawGdocBlockToArchieMLString } from "./rawToArchie.js" import { match, P } from "ts-pattern" -function paragraphToString(paragraph: docs_v1.Schema$Paragraph): string { +function paragraphToString( + paragraph: docs_v1.Schema$Paragraph, + context: { isInList: boolean } +): string { let text = "" - let isInList = false // this is a list const needsBullet = !isNil(paragraph.bullet) - if (needsBullet && !isInList) { - isInList = true + if (needsBullet && !context.isInList) { + context.isInList = true text += `\n[.list]\n` - } else if (!needsBullet && isInList) { - isInList = false + } else if (!needsBullet && context.isInList) { + context.isInList = false text += `[]\n` } @@ -84,6 +86,7 @@ function tableToString( ): string { if (!table) return "" let text = "" + const context = { isInList: false } const { tableRows = [] } = table const rows: RawBlockTableRow[] = [] @@ -104,7 +107,7 @@ function tableToString( const { content = [] } = tableCell for (const item of content) { if (item.paragraph) { - const text = paragraphToString(item.paragraph) + const text = paragraphToString(item.paragraph, context) const rawTextBlock: RawBlockText = { type: "text", value: text, @@ -129,6 +132,7 @@ export async function gdocToArchie( ): Promise<{ text: string }> { // prepare the text holder let text = "" + const context = { isInList: false } // check if the body key and content key exists, and give up if not if (!document.body) return { text } @@ -137,7 +141,7 @@ export async function gdocToArchie( // loop through each content element in the body for (const element of document.body.content) { if (element.paragraph) { - text += paragraphToString(element.paragraph) + text += paragraphToString(element.paragraph, context) } else if (element.table) { text += tableToString(element.table) } @@ -149,7 +153,6 @@ function parseParagraph( element: docs_v1.Schema$ParagraphElement ): Span | RawBlockHorizontalRule | null { // pull out the text - const textRun = element.textRun // sometimes it's not there, skip this all if so diff --git a/site/gdocs/centered-article.scss b/site/gdocs/centered-article.scss index 8d854d9be88..9e139fcba96 100644 --- a/site/gdocs/centered-article.scss +++ b/site/gdocs/centered-article.scss @@ -462,15 +462,10 @@ h3.article-block__heading.has-supertitle { border-collapse: collapse; margin-bottom: 32px; - &.article-block__table--header-column { - th { - text-align: left; - } - } - - &.article-block__table--header-column-row { - th:not([scope="row"]):not([scope="col"]) { - text-align: left; + th { + text-align: left; + &[scope="col"] { + text-align: right; } } From 6daba261f1db1bb4ec786cdcbc4944c3e4309628 Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Mon, 23 Oct 2023 22:17:46 +0000 Subject: [PATCH 06/20] =?UTF-8?q?=F0=9F=94=A8=20remove=20console.log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/model/Gdoc/Gdoc.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/db/model/Gdoc/Gdoc.ts b/db/model/Gdoc/Gdoc.ts index 938e71f5930..2876658e182 100644 --- a/db/model/Gdoc/Gdoc.ts +++ b/db/model/Gdoc/Gdoc.ts @@ -182,7 +182,6 @@ export class Gdoc extends BaseEntity implements OwidGdocInterface { // Convert the doc to ArchieML syntax const { text } = await gdocToArchie(data) - console.log("text", text) // Convert the ArchieML to our enriched JSON structure this.content = archieToEnriched(text) From fd08749dcc7a418157daf6e9df332a97db9ea00f Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Fri, 27 Oct 2023 21:48:19 +0000 Subject: [PATCH 07/20] =?UTF-8?q?=F0=9F=8E=89=20add=20sidebar=20toc=20for?= =?UTF-8?q?=20gdocs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteClient/gdocsDeploy.ts | 1 + db/model/Gdoc/archieToEnriched.ts | 5 ++- .../@ourworldindata/utils/src/owidTypes.ts | 1 + site/css/grid.scss | 1 + site/gdocs/OwidGdoc.tsx | 8 ++++ site/gdocs/centered-article.scss | 40 +++++++++++++++++++ 6 files changed, 54 insertions(+), 2 deletions(-) diff --git a/adminSiteClient/gdocsDeploy.ts b/adminSiteClient/gdocsDeploy.ts index d37d61b2e10..8c83e48884f 100644 --- a/adminSiteClient/gdocsDeploy.ts +++ b/adminSiteClient/gdocsDeploy.ts @@ -59,6 +59,7 @@ export const checkIsLightningUpdate = ( "cover-color": true, "cover-image": true, "hide-citation": true, + "sidebar-toc": true, body: true, dateline: true, details: true, diff --git a/db/model/Gdoc/archieToEnriched.ts b/db/model/Gdoc/archieToEnriched.ts index e9cd4ce04c4..009aecfcfe6 100644 --- a/db/model/Gdoc/archieToEnriched.ts +++ b/db/model/Gdoc/archieToEnriched.ts @@ -131,8 +131,9 @@ function generateToc( if (child.type === "heading") { const { level, text, supertitle } = child const titleString = spansToSimpleString(text) - const supertitleString = - supertitle && spansToSimpleString(supertitle) + const supertitleString = supertitle + ? spansToSimpleString(supertitle) + : "" if (titleString && (level === 2 || level === 3)) { toc.push({ title: titleString, diff --git a/packages/@ourworldindata/utils/src/owidTypes.ts b/packages/@ourworldindata/utils/src/owidTypes.ts index 6e4889d13fb..94edb78a9db 100644 --- a/packages/@ourworldindata/utils/src/owidTypes.ts +++ b/packages/@ourworldindata/utils/src/owidTypes.ts @@ -1324,6 +1324,7 @@ export interface OwidGdocContent { "featured-image"?: string "atom-title"?: string "atom-excerpt"?: string + "sidebar-toc"?: boolean "cover-color"?: | "sdg-color-1" | "sdg-color-2" diff --git a/site/css/grid.scss b/site/css/grid.scss index 98c90e484c6..a8227b71ced 100644 --- a/site/css/grid.scss +++ b/site/css/grid.scss @@ -56,6 +56,7 @@ $grid-responsive: (lg, md, sm); gap: 0 var(--grid-gap); grid-template-columns: repeat(12, 1fr); grid-auto-rows: min-content; + align-items: start; } // Use this when you want a full-page-width grid container diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index 8687373f8ad..4f7e51cfd02 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -21,6 +21,7 @@ import { DebugProvider } from "./DebugContext.js" import { OwidGdocHeader } from "./OwidGdocHeader.js" import StickyNav from "../blocks/StickyNav.js" import { getShortPageCitation } from "./utils.js" +import { TableOfContents } from "../TableOfContents.js" export const AttachmentsContext = createContext<{ linkedCharts: Record linkedDocuments: Record @@ -70,6 +71,7 @@ export function OwidGdoc({ publishedAt ) const citationText = `${shortPageCitation} Published online at OurWorldInData.org. Retrieved from: '${`${BAKED_BASE_URL}/${slug}`}' [Online Resource]` + const hasSidebarToc = content["sidebar-toc"] const bibtex = `@article{owid-${slug.replace(/\//g, "-")}, author = {${formatAuthors({ @@ -119,6 +121,12 @@ export function OwidGdoc({ publishedAt={publishedAt} breadcrumbs={breadcrumbs ?? undefined} /> + {hasSidebarToc && content.toc ? ( + + ) : null} {content.type === "topic-page" && stickyNavLinks?.length ? (