🐝 Ops scripts for Cloudflare Pages (#2711)

* 🐝 move CF deploy scripts from ops repo to owid-grapher

* pull .env from live server

baker/BuildkiteTrigger.ts

@@ -1,8 +1,13 @@
-import { BUILDKITE_API_ACCESS_TOKEN } from "../settings/serverSettings.js"
+import {
+    BUILDKITE_API_ACCESS_TOKEN,
+    BUILDKITE_DEPLOY_CONTENT_PIPELINE_SLUG,
+    BUILDKITE_BRANCH,
+} from "../settings/serverSettings.js"
 
 export class BuildkiteTrigger {
     private organizationSlug = "our-world-in-data"
-    private pipelineSlug = "grapher-cloudflare-pages-deploy-queue"
+    private pipelineSlug = BUILDKITE_DEPLOY_CONTENT_PIPELINE_SLUG
+    private branch = BUILDKITE_BRANCH
 
     async triggerBuild(
         message: string,
@@ -25,7 +30,7 @@ export class BuildkiteTrigger {
 
         const payload = {
             commit: "HEAD",
-            branch: "master",
+            branch: this.branch,
             message: message,
             env: env,
         }

ops/buildkite/README.md

@@ -0,0 +1,14 @@
+## Cloudflare Deployment
+
+Deployment is divided into two main parts:
+
+1. **Code Building**: To avoid race conditions, this process is synchronized.
+2. **Content Baking and Cloudflare Deployment**: This step can run in parallel.
+
+Both processes are triggered by merging to the `master` branch and run through Buildkite pipelines. Additionally, a deploy queue exclusively triggers the content baking step.
+
+**Important Notes**:
+
+-   The scripts operate in `/home/owid/owid-grapher`, not in Buildkite default paths (`$BUILDKITE_BUILD_CHECKOUT_PATH/owid-grapher`).
+-   Run these scripts only on the `master` branch.
+-   For testing on different branches, create a new LXC container (e.g., `owid-lxc create -t owid-cloudflare-prod owid-cloudflare-staging`).

ops/buildkite/build-code

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+#  build-code
+#
+#  Build grapher code.
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+############ WARNING ##############
+# TODO before migrating to ourworldindata.org
+# 1. Change `BAKED_BASE_URL` in this script and in `deploy-content`
+
+# URL of Cloudflare page
+# branches would be deployed to https://[branch].$PROJECT_NAME.pages.dev/"
+# BAKED_BASE_URL="https://$PROJECT_NAME.pages.dev"
+BAKED_BASE_URL="https://pages.owid.io"
+
+build_code () {
+    echo "--- Build code from branch master"
+
+    if [[ "$BRANCH" != "master" ]]; then
+        echo "Error: You're not on the master branch. Exiting."
+        exit 1
+    fi
+
+    update_env
+    update_owid_grapher_repo
+    build_grapher
+
+    echo "--- Code built"
+}
+
+
+update_env() {
+    echo '--- Copying live.owid.io:live/.env to owid-grapher/.env and updating it'
+
+    # copy .env from live server
+    rsync -av --rsh 'ssh -o StrictHostKeyChecking=no' [email protected]:live/.env owid-grapher/.env.template
+
+    # change specific variables
+    # NOTE: `BAKED_SITE_DIR` does not actually change
+    sed -i "s|^BAKED_SITE_DIR=.*$|BAKED_SITE_DIR=/home/owid/live-data/bakedSite|" owid-grapher/.env
+    # NOTE: this should be identical once we merge to master
+    sed -i "s|^BAKED_BASE_URL=.*$|BAKED_BASE_URL=$BAKED_BASE_URL|" owid-grapher/.env
+}
+
+
+update_owid_grapher_repo() {
+    echo '--- Updating owid-grapher'
+    (
+        cd owid-grapher
+        git fetch --all -q
+        git checkout "$BRANCH" -q
+        git reset --hard origin/"$BRANCH"
+    )
+}
+
+build_grapher() {
+    echo '--- Building owid-grapher'
+
+    # NOTE: buildVite creates dist/ folder and `bakeAssets` then copies it to
+    # the bakedSite folder
+    (
+        cd owid-grapher
+        yarn cleanTsc
+        git rev-parse HEAD > /home/owid/live-data/bakedSite/head.txt
+        yarn install
+        yarn buildLerna
+        yarn buildTsc
+        yarn buildVite
+        yarn runDbMigrations
+        node --enable-source-maps --unhandled-rejections=strict itsJustJavascript/baker/algolia/configureAlgolia.js
+        # yarn buildWordpressPlugin
+    )
+}
+
+(
+    echo '==> Acquiring lock'
+    flock -x 200
+
+    build_code
+) 200>/var/lock/build-code.lock

ops/buildkite/deploy-content

@@ -0,0 +1,175 @@
+#!/bin/bash
+#
+#  deploy-content
+#
+#  Bake content and deploy to Cloudflare Pages.
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+# Cloudflare Pages project name
+PROJECT_NAME=owid-grapher
+
+# URL of Cloudflare page
+# branches would be deployed to https://[branch].$PROJECT_NAME.pages.dev/"
+# BAKED_BASE_URL="https://$PROJECT_NAME.pages.dev"
+BAKED_BASE_URL="https://pages.owid.io"
+
+deploy_content () {
+    echo "--- Deploying content to Cloudflare"
+
+    if [[ "$BUILDKITE_BRANCH" != "master" ]]; then
+        echo "Error: You're not on the master branch. Exiting."
+        exit 1
+    fi
+
+    # NOTE: in theory we could run lightning bake in parallel to regular bake and only lock `deploy_to_cloudflare`
+    # right now lightning bake has to wait if there's a regular bake
+    if [ -n "${LIGHTNING_GDOC_SLUGS:-}" ]; then
+        # TODO: do we need to trigger `yarn buildLocalBake --steps gdriveImages` to get up to date images for the lightning post?
+        bake_gdoc_posts "$LIGHTNING_GDOC_SLUGS"
+    else
+        update_owid_content_repo
+        sync_wordpress_uploads
+        bake_site
+        sync_baked_data_to_r2
+    fi
+
+    create_dist
+    deploy_to_cloudflare
+
+    echo "--- Site deployed to $BAKED_BASE_URL"
+}
+
+
+update_owid_content_repo() {
+    echo '--- Updating owid-content'
+    (
+        cd owid-content
+        git fetch --all -q
+        git checkout master -q
+        git reset --hard origin/master
+    )
+}
+
+sync_wordpress_uploads() {
+    echo '--- Syncing live-wordpress uploads from owid-live'
+
+    # see owid-grapher/.../download-wordpress-uploads.sh
+    # this takes about 3 minutes
+    rsync -havzq --delete \
+        --rsh 'ssh -o StrictHostKeyChecking=no' \
+        --exclude='/.gitkeep' \
+        [email protected]:live-data/wordpress/uploads/ wordpress/web/app/uploads
+}
+
+
+sync_to_s3_aws() {
+    local target=$1
+    echo "--- Syncing ${target}..."
+    aws --endpoint=https://nyc3.digitaloceanspaces.com s3 sync live-data/bakedSite/${target} s3://owid-catalog/bake/cloudflare-pages/${target} --acl public-read
+}
+
+sync_to_s3_rclone() {
+    local target=$1
+    echo "--- Syncing ${target}..."
+    rclone sync live-data/bakedSite/${target} spaces-nyc3:owid-catalog/bake/cloudflare-pages/${target} --checkers=64 --ignore-checksum
+}
+
+sync_to_r2_rclone() {
+    local target=$1
+    echo "--- Syncing ${target}..."
+    rclone sync live-data/bakedSite/${target} r2:owid-assets/${target} --checkers=64 --ignore-checksum
+}
+
+sync_to_r2_aws() {
+    local target=$1
+    echo "--- Syncing ${target}..."
+    R2_ENDPOINT_URL=$(grep 'R2_ENDPOINT_URL=' owid-grapher/.env | cut -d '=' -f 2-)
+    aws --profile r2 --endpoint-url "${R2_ENDPOINT_URL}" s3 sync live-data/bakedSite/${target} s3://owid-assets/${target} --acl public-read
+}
+
+sync_baked_data_to_s3() {
+    echo '--- Sync baked data to S3'
+    # Cloudflare Pages has limit of 20000 files
+    sync_to_s3_aws grapher/exports  # 9203 files
+    sync_to_s3_aws exports  # 3314 files
+    sync_to_s3_aws uploads  # 20609 files
+}
+
+sync_baked_data_to_r2() {
+    echo '--- Sync baked data to R2'
+    # Cloudflare Pages has limit of 20000 files
+    # TODO: There's also images/published, which are the gdocs images synced from GDrive.
+    #   There's currently a small-enough amount of them, but we need to sync them to R2 or Cloudflare Images at some point.
+    # NOTE: aws is about 3x faster than rclone
+    sync_to_r2_aws grapher/exports  # 9203 files
+    sync_to_r2_aws exports  # 3314 files
+    sync_to_r2_aws uploads  # 20609 files
+}
+
+deploy_to_cloudflare() {
+    (
+        echo '--- Deploy to Cloudflare'
+        # wrangler uses CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN from owid-grapher/.env, see the token
+        # at https://dash.cloudflare.com/profile/api-tokens
+        # for branch-specific non-production deploys, use `--branch master`
+        cd dist
+        # trim \r, they remove previous logs in buildkite
+        env $(grep -v "^#" ../owid-grapher/.env | xargs -0) npx wrangler pages deploy . --project-name "$PROJECT_NAME" 2>&1 | tr -d '\r'
+    )
+}
+
+create_dist() {
+    echo '--- Creating dist/ folder'
+    # Define a list of excluded directories for rsync
+    EXCLUDES=(grapher/data/variables/ .git/ grapher/exports/ exports/ uploads/)
+
+    # Build rsync command with excluded directories
+    RSYNC_COMMAND=("rsync" "-havzq" "--delete")
+    for EXCLUDE in "${EXCLUDES[@]}"; do
+        RSYNC_COMMAND+=("--exclude=$EXCLUDE")
+    done
+
+    "${RSYNC_COMMAND[@]}" live-data/bakedSite/ dist/
+
+    # remove .etag of gdoc images from dist
+    rm dist/images/published/*.etag
+
+    # copy over Pages Functions as-is
+    # currently superseded by running `deploy` or `dev` inside the owid-grapher dir
+    rsync -havzq --delete owid-grapher/functions dist/
+
+    # we need node_modules for Pages Functions
+    rm -rf dist/node_modules
+    cp -rL owid-grapher/node_modules dist/ # the `L` flag copies over symlinked files, too, which we need for @ourworldindata/utils etc.
+
+    cp owid-grapher/_routes.json dist/_routes.json
+}
+
+
+bake_site() {
+    echo '--- Baking site to ~/live-data/bakedSite'
+
+    (
+        cd owid-grapher
+
+        yarn buildTsc
+        mkdir -p /home/owid/live-data/bakedSite/grapher
+        yarn buildLocalBake $BAKED_BASE_URL /home/owid/live-data/bakedSite
+    )
+}
+
+bake_gdoc_posts() {
+    local slugs="$1"
+    echo "--- Baking GDoc posts ${slugs}"
+    (
+        cd owid-grapher
+        yarn bakeGdocPosts --slugs ${slugs}
+    )
+}
+
+deploy_content

settings/serverSettings.ts

@@ -173,5 +173,10 @@ export const DATA_API_URL: string = clientSettings.DATA_API_URL
 
 export const BUILDKITE_API_ACCESS_TOKEN: string =
     serverSettings.BUILDKITE_API_ACCESS_TOKEN ?? ""
+export const BUILDKITE_DEPLOY_CONTENT_PIPELINE_SLUG: string =
+    serverSettings.BUILDKITE_DEPLOY_CONTENT_PIPELINE_SLUG ||
+    "owid-deploy-content-master"
+export const BUILDKITE_BRANCH: string =
+    serverSettings.BUILDKITE_BRANCH || "master"
 
 export const OPENAI_API_KEY: string = serverSettings.OPENAI_API_KEY ?? ""