From 2bc4806b0ecdab0c2e8e18efb2c5e81dff691a9e Mon Sep 17 00:00:00 2001 From: Rithvik Nishad Date: Tue, 30 Apr 2024 18:03:42 +0530 Subject: [PATCH] Use GH Actions to post EOD activities to slack (#441) --- ...nder.yaml => post-activities-updates.yaml} | 24 +- activities-bot/postUpdates.js | 47 ++++ activities-bot/utils.js | 259 ++++++++++++++++++ .../post-update/[username]/preview/route.ts | 3 +- .../cron/post-update/[username]/route.ts | 44 --- .../slack-eod-bot/cron/post-update/route.ts | 30 +- app/api/slack-eod-bot/eod-updates/route.ts | 18 ++ lib/auth.ts | 4 + lib/slackbotutils.ts | 179 ++---------- scripts/slackEODReminder.js | 148 ---------- 10 files changed, 383 insertions(+), 373 deletions(-) rename .github/workflows/{slack_eod_reminder.yaml => post-activities-updates.yaml} (52%) create mode 100644 activities-bot/postUpdates.js create mode 100644 activities-bot/utils.js delete mode 100644 app/api/slack-eod-bot/cron/post-update/[username]/route.ts create mode 100644 app/api/slack-eod-bot/eod-updates/route.ts create mode 100644 lib/auth.ts delete mode 100644 scripts/slackEODReminder.js diff --git a/.github/workflows/slack_eod_reminder.yaml b/.github/workflows/post-activities-updates.yaml similarity index 52% rename from .github/workflows/slack_eod_reminder.yaml rename to .github/workflows/post-activities-updates.yaml index 57b169f8..ad934832 100644 --- a/.github/workflows/slack_eod_reminder.yaml +++ b/.github/workflows/post-activities-updates.yaml @@ -1,7 +1,22 @@ on: workflow_call: + inputs: + LEADERBOARD_URL: + type: string + required: true + description: URL to the Leaderboard Deployment (to access the EOD API's) + secrets: - SLACK_EOD_BOT_WEBHOOK: + GITHUB_TOKEN: + required: true + + LEADERBOARD_API_KEY: + required: true + + SLACK_EOD_BOT_CHANNEL: + required: true + + SLACK_EOD_BOT_TOKEN: required: true jobs: @@ -15,7 +30,7 @@ jobs: with: repository: coronasafe/leaderboard path: ./leaderboard - sparse-checkout: scripts + sparse-checkout: activities-bot - uses: actions/setup-node@v4 with: @@ -27,4 +42,7 @@ jobs: env: GITHUB_ORG: ${{ github.repository_owner }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_EOD_BOT_WEBHOOK_URL: ${{ secrets.SLACK_EOD_BOT_WEBHOOK }} + LEADERBOARD_URL: ${{ inputs.LEADERBOARD_URL }} + LEADERBOARD_API_KEY: ${{ secrets.LEADERBOARD_API_KEY }} + SLACK_EOD_BOT_CHANNEL: ${{ secrets.SLACK_EOD_BOT_CHANNEL }} + SLACK_EOD_BOT_TOKEN: ${{ secrets.SLACK_EOD_BOT_TOKEN }} diff --git a/activities-bot/postUpdates.js b/activities-bot/postUpdates.js new file mode 100644 index 00000000..24ad59c4 --- /dev/null +++ b/activities-bot/postUpdates.js @@ -0,0 +1,47 @@ +const { + getContributors, + getEvents, + getEODUpdates, + postEODMessage, + mergeUpdates, + flushEODUpdates, +} = require("./utils"); + +async function main() { + const allContributors = await getContributors(); + console.info(`⚙️ Found ${Object.keys(allContributors).length} contributors`); + + console.info("⚙️ Fetching events from GitHub"); + const allEvents = await getEvents(Object.keys(allContributors)); + + console.info("⚙️ Fetching General EOD updates"); + const allEodUpdates = await getEODUpdates(); + + console.info("⚙️ Ready to post EOD updates onto Slack Channel"); + for (const [github, slack] of Object.entries(allContributors)) { + if (github !== "rithviknishad") continue; // TODO: remove this before pushing to prod + + const events = allEvents[github] ?? []; + const eodUpdates = allEodUpdates[github] ?? []; + + const activityCount = events.length + eodUpdates.length; + if (activityCount === 0) { + console.info(`- ⏭️ ${github}: Skipping due to no activity.`); + continue; + } + + await postEODMessage({ + github, + slack, + updates: mergeUpdates(events, eodUpdates), + }); + console.info(`- ✅ ${github}: Posted ${activityCount} updates.`); + } + + // console.info("Flushing EOD updates from cache."); + // await flushEODUpdates(); + + console.info("✅ Completed!"); +} + +main(); diff --git a/activities-bot/utils.js b/activities-bot/utils.js new file mode 100644 index 00000000..161a528d --- /dev/null +++ b/activities-bot/utils.js @@ -0,0 +1,259 @@ +const { readFile, readdir } = require("fs/promises"); +const { join } = require("path"); +const matter = require("gray-matter"); + +const { Octokit } = require("@octokit/action"); +// const { Octokit } = require("octokit"); + +const root = join(process.cwd(), "data-repo/contributors"); + +async function getContributorBySlug(file) { + const { data } = matter(await readFile(join(root, file), "utf8")); + return { + github: file.replace(/\.md$/, ""), + slack: data.slack, + }; +} + +export async function getContributors() { + const slugs = await readdir(`${root}`); + const contributors = await Promise.all( + slugs.map((path) => getContributorBySlug(path)), + ); + + return Object.fromEntries( + contributors + .filter((contributor) => !!contributor.slack) + .map((c) => [c.github, c.slack]), + ); +} + +export const GITHUB_ORG = process.env.GITHUB_ORG; +export const GITHUB_TOKEN = process.env.GITHUB_TOKEN; +export const LEADERBOARD_API_KEY = process.env.LEADERBOARD_API_KEY; +export const LEADERBOARD_URL = process.env.LEADERBOARD_URL; +export const SLACK_EOD_BOT_TOKEN = process.env.SLACK_EOD_BOT_TOKEN; +export const SLACK_EOD_BOT_CHANNEL = process.env.SLACK_EOD_BOT_CHANNEL; + +function isAllowedEvent(event) { + if (event.type === "PullRequestEvent") { + return event.payload.action === "opened"; + } + + if (event.type === "PullRequestReviewEvent") { + return true; + } +} + +export const octokit = new Octokit({ auth: GITHUB_TOKEN }); + +export async function getEvents(allowedAuthors) { + const aDayAgoDate = new Date(); + aDayAgoDate.setDate(aDayAgoDate.getDate() - 1); + const aDayAgo = aDayAgoDate.getTime(); + + const events = await octokit.paginate( + "GET /orgs/{org}/events", + { org: GITHUB_ORG, per_page: 1000 }, + (res) => + res.data.filter( + (event) => + allowedAuthors.includes(event.actor.login) && + isAllowedEvent(event) && + event.created_at && + new Date(event.created_at).getTime() > aDayAgo, + ), + ); + + return Object.groupBy(events, (e) => e.actor.login); +} + +export function mergeUpdates(events, eodUpdates) { + const updates = []; + const counts = { + eod_updates: eodUpdates.length, + pull_requests: 0, + reviews: 0, + }; + + updates.push(...eodUpdates.map((title) => ({ title }))); + + for (const event of events) { + if (event.type === "PullRequestReviewEvent") { + const url = event.payload.pull_request.html_url; + if (!updates.find((a) => a.url === url)) { + counts.reviews += 1; + updates.push({ + title: `Reviewed PR: "${event.payload.pull_request.title}"`, + url, + }); + } + } + + if ( + event.type === "PullRequestEvent" && + event.payload.action === "opened" + ) { + counts.pull_requests += 1; + updates.push({ + title: `Opened PR: "${event.payload.pull_request.title}"`, + url: event.payload.pull_request.html_url, + }); + } + } + + return { updates, counts }; +} + +const leaderboardApiHeaders = { + "Content-Type": "application/json", + Authorization: `${LEADERBOARD_API_KEY}`, +}; + +export async function getEODUpdates() { + const res = await fetch(LEADERBOARD_URL, { + headers: leaderboardApiHeaders, + }); + return res.json(); +} + +export async function flushEODUpdates() { + const res = await fetch(LEADERBOARD_URL, { + headers: leaderboardApiHeaders, + method: "DELETE", + }); + return res.json(); +} + +const slackApiHeaders = { + "Content-Type": "application/json", + Authorization: `Bearer ${SLACK_EOD_BOT_TOKEN}`, +}; + +async function sendSlackMessage(channel, text, blocks) { + const res = await fetch("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: slackApiHeaders, + body: JSON.stringify({ + // channel, + channel: "U02TDGQQPMJ", // TODO: replace with channel before pushign + text, + ...blocks, + }), + }); + + const data = await res.json(); + if (!data.ok) { + console.error(data); + } +} + +export function getHumanReadableUpdates( + { updates, counts }, + slackID, + githubId, +) { + const colorRange = [ + { color: "#00FF00", min: 5 }, + { color: "#FFFF00", min: 1 }, + { color: "#FF0000", min: 0 }, + ]; + + const activityCount = + counts.pull_requests + counts.reviews + counts.eod_updates; + + const color = + colorRange.find((range) => range.min <= activityCount)?.color || "#0000FF"; + + return { + attachments: [ + { + color, + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: ` +*EOD Updates for <@${slackID}>* + +Summary: Opened *${counts.pull_requests}* pull requests, Reviewed *${counts.reviews}* PRs and *${counts.eod_updates}* other general updates. + + | <${process.env.NEXT_PUBLIC_META_URL}/contributors/${githubId}|Contributor Profile> +`, + }, + accessory: { + type: "image", + image_url: `https://avatars.githubusercontent.com/${githubId}?s=128`, + alt_text: "profile image", + }, + }, + + { + type: "divider", + }, + { + type: "rich_text", + elements: [ + { + type: "rich_text_section", + elements: [ + { + type: "text", + text: "Updates:", + style: { bold: true }, + }, + { + type: "text", + text: `\n${updates.length === 0 ? "No updates for today" : ""}`, + }, + ], + }, + { + type: "rich_text_list", + style: "bullet", + elements: updates.map((item) => { + const elements = [ + { + type: "text", + text: item.title, + }, + ]; + + if (item.url) { + let preview = ""; + + if (item.url.startsWith("https://github.com")) { + preview = item.url.replace("https://github.com/", ""); + const [org, repo, type, number] = preview.split("/"); + preview = `${repo}#${number.split("#")[0]}`; + } + + elements.push({ + type: "link", + text: ` ${preview} ↗`, + url: item.url, + }); + } + + return { + type: "rich_text_section", + elements, + }; + }), + }, + ], + }, + ], + }, + ], + }; +} + +export async function postEODMessage({ github, slack, updates }) { + await sendSlackMessage( + SLACK_EOD_BOT_CHANNEL, + "", + getHumanReadableUpdates(updates, slack, github), + ); +} diff --git a/app/api/slack-eod-bot/cron/post-update/[username]/preview/route.ts b/app/api/slack-eod-bot/cron/post-update/[username]/preview/route.ts index ef724161..a17454ec 100644 --- a/app/api/slack-eod-bot/cron/post-update/[username]/preview/route.ts +++ b/app/api/slack-eod-bot/cron/post-update/[username]/preview/route.ts @@ -1,11 +1,10 @@ -import { NextRequest } from "next/server"; import { EODUpdatesManager, sendSlackMessage } from "@/lib/slackbotutils"; import { getContributorBySlug } from "@/lib/api"; export const maxDuration = 300; export async function GET( - req: NextRequest, + req: Request, { params }: { params: { username: string } }, ) { const { username } = params; diff --git a/app/api/slack-eod-bot/cron/post-update/[username]/route.ts b/app/api/slack-eod-bot/cron/post-update/[username]/route.ts deleted file mode 100644 index f0a7c688..00000000 --- a/app/api/slack-eod-bot/cron/post-update/[username]/route.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { NextRequest } from "next/server"; -import { - EODUpdatesManager, - getHumanReadableUpdates, - sendSlackMessage, -} from "@/lib/slackbotutils"; -import { getDailyReport } from "@/lib/contributor"; -import { getContributorBySlug } from "@/lib/api"; - -export const maxDuration = 300; - -export async function POST( - req: NextRequest, - { params }: { params: { username: string } }, -) { - const { username } = params; - const { reviews } = await req.json(); - - if ( - process.env.CRON_SECRET && - req.headers.get("Authorization") !== `Bearer ${process.env.CRON_SECRET}` - ) { - return new Response("Unauthorized", { status: 401 }); - } - - const contributor = await getContributorBySlug(`${username}.md`); - if (!contributor?.slack) { - return new Response("Contributor not found", { status: 404 }); - } - - const eodUpdatesManager = EODUpdatesManager(contributor); - const report = await getDailyReport(username, reviews); - const eodUpdates = await eodUpdatesManager.get(); - - const updates = getHumanReadableUpdates( - report, - eodUpdates, - contributor.slack, - ); - - await sendSlackMessage(process.env.SLACK_EOD_BOT_CHANNEL || "", "", updates); - eodUpdatesManager.clear(); - return new Response("OK"); -} diff --git a/app/api/slack-eod-bot/cron/post-update/route.ts b/app/api/slack-eod-bot/cron/post-update/route.ts index cd953308..5a3d69f3 100644 --- a/app/api/slack-eod-bot/cron/post-update/route.ts +++ b/app/api/slack-eod-bot/cron/post-update/route.ts @@ -1,11 +1,7 @@ import { getContributors } from "@/lib/api"; -import { getPullRequestReviews } from "@/lib/contributor"; import { NextRequest } from "next/server"; export const GET = async (req: NextRequest) => { - const params = req.nextUrl.searchParams; - const preview = params.get("preview") === "yes"; - if ( process.env.CRON_SECRET && req.headers.get("Authorization") !== `Bearer ${process.env.CRON_SECRET}` @@ -19,27 +15,11 @@ export const GET = async (req: NextRequest) => { Authorization: `Bearer ${process.env.CRON_SECRET}`, }; - if (preview) { - for (const user of users) { - fetch( - `https://contributors.ohc.network/api/slack-eod-bot/cron/post-update/${user.github}/preview`, - { method: "GET", headers }, - ); - } - } else { - const reviews = await getPullRequestReviews(); - for (const user of users) { - fetch( - `https://contributors.ohc.network/api/slack-eod-bot/cron/post-update/${user.github}`, - { - method: "POST", - headers, - body: JSON.stringify({ - reviews: reviews.filter((r) => r.author === user.github), - }), - }, - ); - } + for (const user of users) { + fetch( + `https://contributors.ohc.network/api/slack-eod-bot/cron/post-update/${user.github}/preview`, + { method: "GET", headers }, + ); } return new Response("OK"); diff --git a/app/api/slack-eod-bot/eod-updates/route.ts b/app/api/slack-eod-bot/eod-updates/route.ts new file mode 100644 index 00000000..d12233c6 --- /dev/null +++ b/app/api/slack-eod-bot/eod-updates/route.ts @@ -0,0 +1,18 @@ +import { isAuthenticated } from "@/lib/auth"; +import { clearAllEODUpdates, getAllEODUpdates } from "@/lib/slackbotutils"; + +export const GET = async (req: Request) => { + if (!isAuthenticated(req)) { + return Response.json({ error: "Unauthorized" }, { status: 403 }); + } + const updates = await getAllEODUpdates(); + return Response.json(updates); +}; + +export const DELETE = async (req: Request) => { + if (!isAuthenticated(req)) { + return Response.json({ error: "Unauthorized" }, { status: 403 }); + } + const count = await clearAllEODUpdates(); + return Response.json({ count }); +}; diff --git a/lib/auth.ts b/lib/auth.ts new file mode 100644 index 00000000..93d21d97 --- /dev/null +++ b/lib/auth.ts @@ -0,0 +1,4 @@ +export const isAuthenticated = (req: Request) => { + const API_KEY = process.env.LEADERBOARD_API_KEY; + return API_KEY ? req.headers.get("Authorization") === API_KEY : true; +}; diff --git a/lib/slackbotutils.ts b/lib/slackbotutils.ts index da63b87f..db849586 100644 --- a/lib/slackbotutils.ts +++ b/lib/slackbotutils.ts @@ -1,159 +1,8 @@ import { kv } from "@vercel/kv"; import { formatDuration as _formatDuration } from "date-fns"; import { getDailyReport } from "./contributor"; -import { getContributors } from "@/lib/api"; import { Contributor } from "@/lib/types"; -export const getHumanReadableUpdates = ( - data: Awaited>, - generalUpdates: string[], - slackID: string, -) => { - const sections = [ - { - title: `Pull Requests Opened`, - count: data.pull_requests.length, - items: data.pull_requests.map((pr) => ({ - title: pr.title, - url: pr.url, - })), - }, - { - title: `Commits`, - count: data.commits.length, - items: data.commits.map((commit) => ({ - title: commit.title, - url: commit.url, - })), - }, - { - title: `Reviews`, - count: data.reviews.length, - items: data.reviews.map((review) => ({ - title: review.pull_request, - url: review.url, - })), - }, - { - title: `General updates`, - count: generalUpdates.length, - items: generalUpdates.map((title) => ({ title, url: undefined })), - }, - { - title: `Active Issues`, - count: data.issues_active.length, - items: data.issues_active.map((issue) => ({ - title: issue.title, - url: issue.url, - })), - }, - { - title: `Pending Issues`, - count: data.issues_pending.length, - items: data.issues_pending.map((issue) => ({ - title: issue.title, - url: issue.url, - })), - }, - ]; - - const colorRange = [ - { - color: "#00FF00", - min: 5, - }, - { - color: "#FFFF00", - min: 1, - }, - { - color: "#FF0000", - min: 0, - }, - ]; - - const color = - colorRange.find( - (range) => - range.min <= - data.pull_requests.length + data.commits.length + data.reviews.length, - )?.color || "#0000FF"; - - return { - attachments: [ - { - color, - blocks: [ - { - type: "header", - text: { - type: "plain_text", - text: `Updates for ${data.user_info.data.name || data.user_info.data.url.split("/").at(-1)}`, - }, - }, - { - type: "section", - text: { - type: "mrkdwn", - text: `_${data.user_info.data.url.split("/").at(-1)}_\n<@${slackID}>\n<${data.user_info.data.url}|Github Profile>\n<${process.env.NEXT_PUBLIC_META_URL}/contributors/${data.user_info.data.url.split("/").at(-1)}|Contributor Profile>`, - }, - accessory: { - type: "image", - image_url: data.user_info.data.avatar_url, - alt_text: "profile image", - }, - }, - ...sections.flatMap((section) => [ - { - type: "divider", - }, - { - type: "rich_text", - elements: [ - { - type: "rich_text_section", - elements: [ - { - type: "text", - text: `${section.title} (${section.count})`, - style: { - bold: true, - }, - }, - { - type: "text", - text: `\n${section.items.length === 0 ? `No ${section.title.toLowerCase()} for today` : ""}`, - }, - ], - }, - { - type: "rich_text_list", - style: "bullet", - elements: section.items.map((item) => ({ - type: "rich_text_section", - elements: [ - item.url - ? { - type: "link", - text: item.title, - url: item.url, - } - : { - type: "text", - text: item.title, - }, - ], - })), - }, - ], - }, - ]), - ], - }, - ], - }; -}; - const slackApiHeaders = { "Content-Type": "application/json", Authorization: `Bearer ${process.env.SLACK_EOD_BOT_TOKEN}`, @@ -206,6 +55,34 @@ export const reactToMessage = async ( } }; +const getAllEODUpdatesKeys = async () => { + const keys: string[] = []; + for await (const key of kv.scanIterator({ match: "eod:*", count: 100 })) { + keys.push(key); + } + return keys; +}; + +export const getAllEODUpdates = async () => { + const userUpdates: Record = {}; + + const keys = await getAllEODUpdatesKeys(); + + if (keys.length) { + const values: string[][] = await kv.mget(...keys); + for (let i = 0; i < keys.length; i++) { + userUpdates[keys[i].replace("eod:", "")] = Array.from(new Set(values[i])); + } + } + + return userUpdates; +}; + +export const clearAllEODUpdates = async () => { + const keys = await getAllEODUpdatesKeys(); + return kv.del(...keys); +}; + const getEODUpdates = async (contributor: Contributor) => { return ((await kv.get("eod:" + contributor.github)) || []) as string[]; }; diff --git a/scripts/slackEODReminder.js b/scripts/slackEODReminder.js deleted file mode 100644 index 9e223edd..00000000 --- a/scripts/slackEODReminder.js +++ /dev/null @@ -1,148 +0,0 @@ -const { Octokit } = require("@octokit/action"); -// const { Octokit } = require("octokit"); // Use this for local development -const matter = require("gray-matter"); -const { readFile, readdir } = require("fs/promises"); -const { join } = require("path"); - -const root = join(process.cwd(), "data-repo/contributors"); - -const GITHUB_ORG = process.env.GITHUB_ORG; -const GITHUB_TOKEN = process.env.GITHUB_TOKEN; -const SLACK_WEBHOOK_URL = process.env.SLACK_EOD_BOT_WEBHOOK_URL; - -const octokit = new Octokit({ - auth: GITHUB_TOKEN, -}); - -async function getContributorBySlug(file) { - const { data } = matter(await readFile(join(root, file), "utf8")); - return { - github: file.replace(/\.md$/, ""), - slack: data.slack, - }; -} - -async function getContributors() { - const slugs = await readdir(`${root}`); - return Promise.all(slugs.map((path) => getContributorBySlug(path))); -} - -const isPROpenEvent = (event) => { - return event.type === "PullRequestEvent" && event.payload.action === "opened"; -}; - -const isReviewEvent = (event) => { - return event.type === "PullRequestReviewEvent"; -}; - -const isIssuePendingEvent = (event) => { - return ( - event.type === "IssuesEvent" && - event.payload.action === "assigned" && - event.payload.issue.state === "open" - ); -}; - -async function fetchGitHubEvents(authors) { - const aDayAgoDate = new Date(); - aDayAgoDate.setDate(aDayAgoDate.getDate() - 1); - const aDayAgo = aDayAgoDate.getTime(); - - const events = await octokit.paginate( - "GET /orgs/{org}/events", - { - org: GITHUB_ORG, - per_page: 1000, - }, - (res) => - res.data.filter( - (event) => - authors.includes(event.actor.login) && - event.created_at && - new Date(event.created_at).getTime() > aDayAgo && - (isPROpenEvent(event) || - isReviewEvent(event) || - isIssuePendingEvent(event)), - ), - ); - - return events; -} - -function summarizeActivity(events) { - const activities = events - .map((event) => { - if (isPROpenEvent(event)) { - return `- Made a PR: ${event.payload.pull_request.title} : ${event.payload.pull_request.html_url}`; - } - - return null; - }) - .filter(Boolean); - - const reviewEvents = events.filter(isReviewEvent); - const reviewsDistinctOnPRs = Object.keys( - Object.groupBy(reviewEvents, (event) => event.payload.pull_request.id), - ).length; - if (reviewsDistinctOnPRs) { - activities.push(`- Reviewed ${reviewsDistinctOnPRs} pull request(s).`); - } - - const upcoming = events - .map((event) => { - if (isIssuePendingEvent(event)) { - return `- Work on: ${event.payload.issue?.title} : ${event.payload.issue.html_url}`; - } - - return null; - }) - .filter(Boolean); - - return { - activities: activities.join("\n") || "None", - upcoming_activities: upcoming.join("\n") || "None", - }; -} - -async function main() { - const today = new Date(); - const eod_date = `${today.getDate()}/${today.getMonth()}/${today.getFullYear()}`; - - const members = (await getContributors()).filter((c) => c.slack); - const events = await fetchGitHubEvents(members.map((c) => c.github)); - - const memberEvents = members.map((c) => ({ - slack: c.slack, - github: c.github, - events: events.filter((e) => e.actor.login === c.github), - })); - - const status = await Promise.all( - memberEvents.map(async (member) => { - const payload = { - user_id: member.slack, - eod_date, - ...summarizeActivity(member.events), - }; - - const res = await fetch(SLACK_WEBHOOK_URL, { - method: "POST", - body: JSON.stringify(payload), - }); - - const data = await res.json(); - - return { - slack: member.slack, - github: member.github, - payload, - webhook_response: { - status: res.status, - data: data, - }, - }; - }), - ); -} - -main();