From 1fb9cdffff1a53239ba3be1feaa7165c98a76a35 Mon Sep 17 00:00:00 2001 From: Rithvik Nishad Date: Thu, 14 Nov 2024 19:25:30 +0530 Subject: [PATCH 1/5] Adds support for showing summary in leaderboard (#544) --- app/leaderboard/[duration]/Leaderboard.tsx | 68 ++++++++++++++++++++++ lib/api.ts | 7 +++ lib/types.ts | 2 + scraper/src/github-scraper/types.ts | 1 + 4 files changed, 78 insertions(+) diff --git a/app/leaderboard/[duration]/Leaderboard.tsx b/app/leaderboard/[duration]/Leaderboard.tsx index bf27a896..09e6d5d8 100644 --- a/app/leaderboard/[duration]/Leaderboard.tsx +++ b/app/leaderboard/[duration]/Leaderboard.tsx @@ -20,6 +20,9 @@ import { HiSortAscending, HiSortDescending } from "react-icons/hi"; import { Popover } from "@headlessui/react"; import Link from "next/link"; import { useRouter, useSearchParams } from "next/navigation"; +import { BiGitPullRequest } from "react-icons/bi"; +import { GoIssueOpened, GoIssueClosed } from "react-icons/go"; +import { VscGitPullRequestClosed } from "react-icons/vsc"; const filterBySearchTerm = (searchTermLC: string) => { return (item: LeaderboardAPIResponse[number]) => @@ -221,6 +224,71 @@ export default function Leaderboard(props: Props) { + {/* Stats Summary */} +
+
+
+ +
+

+ Issues Opened +

+

+ {resultSet.reduce( + (sum, user) => sum + user.highlights.issue_opened, + 0, + )} +

+
+
+ +
+ +
+

+ Issues Closed +

+

+ {resultSet.reduce( + (sum, user) => sum + user.highlights.issue_closed, + 0, + )} +

+
+
+ +
+ +
+

+ PRs Opened +

+

+ {resultSet.reduce( + (sum, user) => sum + user.highlights.pr_opened, + 0, + )} +

+
+
+ +
+ +
+

+ PRs Merged +

+

+ {resultSet.reduce( + (sum, user) => sum + user.highlights.pr_merged, + 0, + )} +

+
+
+
+
+ {/* Leaderboard List */}
diff --git a/lib/api.ts b/lib/api.ts index 58806ad1..01568e29 100644 --- a/lib/api.ts +++ b/lib/api.ts @@ -147,6 +147,8 @@ export async function getContributorBySlug(file: string, detail = false) { acc.issue_assigned + (activity.type === "issue_assigned" ? 1 : 0), issue_opened: acc.issue_opened + (activity.type === "issue_opened" ? 1 : 0), + issue_closed: + acc.issue_closed + (activity.type === "issue_closed" ? 1 : 0), discussion_created: acc.discussion_created + (activity.type === "discussion_created" ? 2 : 0), @@ -169,6 +171,7 @@ export async function getContributorBySlug(file: string, detail = false) { pr_reviewed: 0, issue_assigned: 0, issue_opened: 0, + issue_closed: 0, discussion_created: 0, discussion_answered: 0, discussion_comment_created: 0, @@ -209,6 +212,7 @@ export async function getContributorBySlug(file: string, detail = false) { pr_collaborated: weightedActivity.pr_collaborated, issue_assigned: weightedActivity.issue_assigned, issue_opened: weightedActivity.issue_opened, + issue_closed: weightedActivity.issue_closed, discussion_created: weightedActivity.discussion_created, discussion_answered: weightedActivity.discussion_answered, discussion_comment_created: weightedActivity.discussion_comment_created, @@ -288,6 +292,7 @@ const HIGHLIGHT_KEYS = [ "pr_collaborated", "issue_assigned", "issue_opened", + "issue_closed", "discussion_created", "discussion_answered", "discussion_comment_created", @@ -313,6 +318,7 @@ const HighlightsReducer = (acc: Highlights, day: Highlights) => { pr_collaborated: acc.pr_collaborated + (day.pr_collaborated ?? 0), issue_assigned: acc.issue_assigned + (day.issue_assigned ?? 0), issue_opened: acc.issue_opened + (day.issue_opened ?? 0), + issue_closed: acc.issue_closed + (day.issue_closed ?? 0), discussion_created: acc.discussion_created + (day.discussion_created ?? 0), discussion_answered: acc.discussion_answered + (day.discussion_answered ?? 0), @@ -331,6 +337,7 @@ const HighlightsInitialValue = { pr_collaborated: 0, issue_assigned: 0, issue_opened: 0, + issue_closed: 0, discussion_created: 0, discussion_answered: 0, discussion_comment_created: 0, diff --git a/lib/types.ts b/lib/types.ts index 09a3b755..2e9ad27a 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -43,6 +43,7 @@ export interface Highlights { pr_collaborated: number; issue_assigned: number; issue_opened: number; + issue_closed: number; pr_stale?: number; discussion_created: number; discussion_answered: number; @@ -59,6 +60,7 @@ export interface WeekSummary { pr_collaborated: number; issue_assigned: number; issue_opened: number; + issue_closed: number; discussion_created: number; discussion_answered: number; discussion_comment_created: number; diff --git a/scraper/src/github-scraper/types.ts b/scraper/src/github-scraper/types.ts index 942c54cc..4da93cf4 100644 --- a/scraper/src/github-scraper/types.ts +++ b/scraper/src/github-scraper/types.ts @@ -135,6 +135,7 @@ export const ACTIVITY_TYPES = [ "issue_closed", "pr_reviewed", "issue_opened", + "issue_closed", "eod_update", "pr_opened", "pr_merged", From 700b2ca8d0616816a17c6aacc20cda2e69760e6e Mon Sep 17 00:00:00 2001 From: Pooranjoy Bhattacharya Date: Wed, 20 Nov 2024 12:21:22 +0530 Subject: [PATCH 2/5] Caching profile activity filters using `atomWithStorage` (#547) Co-authored-by: rithviknishad --- components/contributors/GithubActivity.tsx | 98 +++------------------- package.json | 1 + pnpm-lock.yaml | 26 +++++- 3 files changed, 35 insertions(+), 90 deletions(-) diff --git a/components/contributors/GithubActivity.tsx b/components/contributors/GithubActivity.tsx index 11bc61b9..a75e2062 100644 --- a/components/contributors/GithubActivity.tsx +++ b/components/contributors/GithubActivity.tsx @@ -7,7 +7,6 @@ import { parseOrgRepoFromURL, } from "@/lib/utils"; import OpenGraphImage from "../gh_events/OpenGraphImage"; -import { useState } from "react"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; import RelativeTime from "../RelativeTime"; import DateRangePicker from "../DateRangePicker"; @@ -15,6 +14,12 @@ import { format } from "date-fns"; import GithubDiscussion from "../discussions/GithubDiscussion"; import { IoIosChatboxes } from "react-icons/io"; import Link from "next/link"; +import { atomWithStorage } from "jotai/utils"; +import { useAtom } from "jotai"; + +const activityTypesAtom = atomWithStorage("leaderboard-activity-types", [ + ...ACTIVITY_TYPES, +]); let commentTypes = (activityEvent: string[]) => { switch (activityEvent[0]) { @@ -339,30 +344,6 @@ const activitiesOfType = (types: Activity["type"][]) => { }; }; -/*const getRangeFilterPresets = (activities: Activity[]) => { - if (!activities.length) return []; - - const latest = new Date(activities[0].time); - let oldest = new Date(latest); - - activities.forEach((activity) => { - const time = new Date(activity.time); - if (time < oldest) { - oldest = time; - } - }); - - let current = new Date(oldest.getFullYear(), oldest.getMonth()); - const end = new Date(latest.getFullYear(), latest.getMonth()); - - const results: string[] = []; - while (current <= end) { - results.push(current.toISOString().slice(0, 7)); - current.setMonth(current.getMonth() + 1); - } - return results.reverse(); -};*/ - interface Props { activityData: ActivityData; } @@ -371,7 +352,6 @@ export default function GithubActivity({ activityData }: Props) { const router = useRouter(); const pathname = usePathname(); const searchParams = useSearchParams(); - // const rangeQuery = searchParams.get("range") ?? "last-month"; const [start, end] = parseDateRangeSearchParam(searchParams.get("between")); const updateSearchParam = (key: string, value?: string) => { @@ -386,32 +366,7 @@ export default function GithubActivity({ activityData }: Props) { router.replace(`${pathname}${query}`, { scroll: false }); }; - const [activityTypes, setActivityTypes] = useState([...ACTIVITY_TYPES]); - - /*const range = useMemo(() => { - const to = new Date(); - to.setDate(to.getDate() + 1); - - if (rangeQuery === "last-month") { - const from = new Date(to); - from.setDate(from.getDate() - 30); - return { from, to: to }; - } else if (rangeQuery === "last-week") { - const from = new Date(to); - from.setDate(from.getDate() - 7); - return { from, to }; - } else { - const from = new Date(rangeQuery); - const to = new Date(rangeQuery); - to.setMonth(to.getMonth() + 1); - return { from, to }; - } - }, [rangeQuery]);*/ - - /*const rangePresets = useMemo( - () => getRangeFilterPresets(activityData["activity"]), - [activityData], - );*/ + const [activityTypes, setActivityTypes] = useAtom(activityTypesAtom); const activitiesInRange = activityData.activity.filter( activitiesBetween({ from: start, to: end }), @@ -437,36 +392,6 @@ export default function GithubActivity({ activityData }: Props) { ); }} /> - {/* */} {ACTIVITY_TYPES.map((type) => ( { - const final = event.target.checked - ? Array.from(new Set([...props.state, props.type])) - : props.state.filter((type) => type !== props.type); - - props.setState(final); + const isChecked = event.target.checked; + const final = new Set([...props.state]); + final[isChecked ? "add" : "delete"](props.type); + props.setState(Array.from(final)); }} />{" "} { diff --git a/package.json b/package.json index dc32c52e..93c2c6f1 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "clsx": "^1.2.1", "date-fns": "^2.30.0", "gray-matter": "^4.0.3", + "jotai": "^2.10.2", "next": "^14.2.10", "next-themes": "^0.2.1", "react": "^18.3.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5311b61d..c6d41083 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: gray-matter: specifier: ^4.0.3 version: 4.0.3 + jotai: + specifier: ^2.10.2 + version: 2.10.2(@types/react@18.3.3)(react@18.3.1) next: specifier: ^14.2.10 version: 14.2.10(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -1467,6 +1470,18 @@ packages: resolution: {integrity: sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==} hasBin: true + jotai@2.10.2: + resolution: {integrity: sha512-DqsBTlRglIBviuJLfK6JxZzpd6vKfbuJ4IqRCz70RFEDeZf46Fcteb/FXxNr1UnoxR5oUy3oq7IE8BrEq0G5DQ==} + engines: {node: '>=12.20.0'} + peerDependencies: + '@types/react': '>=17.0.0' + react: '>=17.0.0' + peerDependenciesMeta: + '@types/react': + optional: true + react: + optional: true + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -3465,7 +3480,7 @@ snapshots: eslint: 8.16.0 eslint-import-resolver-node: 0.3.9 eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.16.0))(eslint@8.16.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.16.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.16.0))(eslint@8.16.0))(eslint@8.16.0) eslint-plugin-jsx-a11y: 6.9.0(eslint@8.16.0) eslint-plugin-react: 7.35.0(eslint@8.16.0) eslint-plugin-react-hooks: 4.6.2(eslint@8.16.0) @@ -3493,7 +3508,7 @@ snapshots: enhanced-resolve: 5.17.1 eslint: 8.16.0 eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.16.0))(eslint@8.16.0))(eslint@8.16.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.16.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.16.0))(eslint@8.16.0))(eslint@8.16.0) fast-glob: 3.3.2 get-tsconfig: 4.7.6 is-core-module: 2.15.0 @@ -3515,7 +3530,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.16.0): + eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.16.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.16.0))(eslint@8.16.0))(eslint@8.16.0): dependencies: array-includes: 3.1.8 array.prototype.findlastindex: 1.2.5 @@ -4098,6 +4113,11 @@ snapshots: jiti@1.21.6: {} + jotai@2.10.2(@types/react@18.3.3)(react@18.3.1): + optionalDependencies: + '@types/react': 18.3.3 + react: 18.3.1 + js-tokens@4.0.0: {} js-yaml@3.14.1: From 8924973215a3449b4de4778a9b0ea753b0cfd95d Mon Sep 17 00:00:00 2001 From: Rithvik Nishad Date: Thu, 28 Nov 2024 20:55:21 +0530 Subject: [PATCH 3/5] gh scraper: improve logging and scrape everything (beyond configured date range) (#549) --- package.json | 3 + pnpm-lock.yaml | 105 ++++++++++++++++++++++ scraper/package.json | 5 +- scraper/pnpm-lock.yaml | 40 ++++++--- scraper/src/github-scraper/config.ts | 8 +- scraper/src/github-scraper/fetchEvents.ts | 92 ++++++++++++------- scraper/src/github-scraper/index.ts | 18 ++-- scraper/src/github-scraper/parseEvents.ts | 4 +- scraper/tsconfig.json | 18 ++-- 9 files changed, 225 insertions(+), 68 deletions(-) diff --git a/package.json b/package.json index 93c2c6f1..945054a3 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,9 @@ }, "dependencies": { "@headlessui/react": "^1.7.19", + "@octokit/core": "^6.1.2", + "@octokit/plugin-paginate-graphql": "^5.2.4", + "@octokit/plugin-paginate-rest": "^11.3.6", "@t3-oss/env-nextjs": "^0.9.2", "@vercel/kv": "^1.0.1", "clsx": "^1.2.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c6d41083..e3436367 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,15 @@ importers: '@headlessui/react': specifier: ^1.7.19 version: 1.7.19(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@octokit/core': + specifier: ^6.1.2 + version: 6.1.2 + '@octokit/plugin-paginate-graphql': + specifier: ^5.2.4 + version: 5.2.4(@octokit/core@6.1.2) + '@octokit/plugin-paginate-rest': + specifier: ^11.3.6 + version: 11.3.6(@octokit/core@6.1.2) '@t3-oss/env-nextjs': specifier: ^0.9.2 version: 0.9.2(typescript@5.5.4)(zod@3.23.8) @@ -290,6 +299,10 @@ packages: resolution: {integrity: sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==} engines: {node: '>= 18'} + '@octokit/auth-token@5.1.1': + resolution: {integrity: sha512-rh3G3wDO8J9wSjfI436JUKzHIxq8NaiL0tVeB2aXmG6p/9859aUOAjA9pmSPNGGZxfwmaJ9ozOJImuNVJdpvbA==} + engines: {node: '>= 18'} + '@octokit/auth-unauthenticated@5.0.1': resolution: {integrity: sha512-oxeWzmBFxWd+XolxKTc4zr+h3mt+yofn4r7OfoIkR/Cj/o70eEGmPsFbueyJE2iBAGpjgTnEOKM3pnuEGVmiqg==} engines: {node: '>= 18'} @@ -298,6 +311,14 @@ packages: resolution: {integrity: sha512-1LFfa/qnMQvEOAdzlQymH0ulepxbxnCYAKJZfMci/5XJyIHWgEYnDmgnKakbTh7CH2tFQ5O60oYDvns4i9RAIg==} engines: {node: '>= 18'} + '@octokit/core@6.1.2': + resolution: {integrity: sha512-hEb7Ma4cGJGEUNOAVmyfdB/3WirWMg5hDuNFVejGEDFqupeOysLc2sG6HJxY2etBp5YQu5Wtxwi020jS9xlUwg==} + engines: {node: '>= 18'} + + '@octokit/endpoint@10.1.1': + resolution: {integrity: sha512-JYjh5rMOwXMJyUpj028cu0Gbp7qe/ihxfJMLc8VZBMMqSwLgOxDI1911gV4Enl1QSavAQNJcwmwBF9M0VvLh6Q==} + engines: {node: '>= 18'} + '@octokit/endpoint@9.0.5': resolution: {integrity: sha512-ekqR4/+PCLkEBF6qgj8WqJfvDq65RH85OAgrtnVp1mSxaXF03u2xW/hUdweGS5654IlC0wkNYC18Z50tSYTAFw==} engines: {node: '>= 18'} @@ -306,6 +327,10 @@ packages: resolution: {integrity: sha512-r+oZUH7aMFui1ypZnAvZmn0KSqAUgE1/tUXIWaqUCa1758ts/Jio84GZuzsvUkme98kv0WFY8//n0J1Z+vsIsQ==} engines: {node: '>= 18'} + '@octokit/graphql@8.1.1': + resolution: {integrity: sha512-ukiRmuHTi6ebQx/HFRCXKbDlOh/7xEV6QUXaE7MJEKGNAncGI/STSbOkl12qVXZrfZdpXctx5O9X1AIaebiDBg==} + engines: {node: '>= 18'} + '@octokit/oauth-app@6.1.0': resolution: {integrity: sha512-nIn/8eUJ/BKUVzxUXd5vpzl1rwaVxMyYbQkNZjHrF7Vk/yu98/YDF/N2KeWO7uZ0g3b5EyiFXFkZI8rJ+DH1/g==} engines: {node: '>= 18'} @@ -330,12 +355,24 @@ packages: peerDependencies: '@octokit/core': '>=5' + '@octokit/plugin-paginate-graphql@5.2.4': + resolution: {integrity: sha512-pLZES1jWaOynXKHOqdnwZ5ULeVR6tVVCMm+AUbp0htdcyXDU95WbkYdU4R2ej1wKj5Tu94Mee2Ne0PjPO9cCyA==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '>=6' + '@octokit/plugin-paginate-rest@11.3.1': resolution: {integrity: sha512-ryqobs26cLtM1kQxqeZui4v8FeznirUsksiA+RYemMPJ7Micju0WSkv50dBksTuZks9O5cg4wp+t8fZ/cLY56g==} engines: {node: '>= 18'} peerDependencies: '@octokit/core': '5' + '@octokit/plugin-paginate-rest@11.3.6': + resolution: {integrity: sha512-zcvqqf/+TicbTCa/Z+3w4eBJcAxCFymtc0UAIsR3dEVoNilWld4oXdscQ3laXamTszUZdusw97K8+DrbFiOwjw==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '>=6' + '@octokit/plugin-paginate-rest@9.2.1': resolution: {integrity: sha512-wfGhE/TAkXZRLjksFXuDZdmGnJQHvtU/joFQdweXUgzo1XwvBCD4o4+75NtFfjfLK5IwLf9vHTfSiU3sLRYpRw==} engines: {node: '>= 18'} @@ -364,16 +401,27 @@ packages: resolution: {integrity: sha512-GETXfE05J0+7H2STzekpKObFe765O5dlAKUTLNGeH+x47z7JjXHfsHKo5z21D/o/IOZTUEI6nyWyR+bZVP/n5Q==} engines: {node: '>= 18'} + '@octokit/request-error@6.1.5': + resolution: {integrity: sha512-IlBTfGX8Yn/oFPMwSfvugfncK2EwRLjzbrpifNaMY8o/HTEAFqCA1FZxjD9cWvSKBHgrIhc4CSBIzMxiLsbzFQ==} + engines: {node: '>= 18'} + '@octokit/request@8.4.0': resolution: {integrity: sha512-9Bb014e+m2TgBeEJGEbdplMVWwPmL1FPtggHQRkV+WVsMggPtEkLKPlcVYm/o8xKLkpJ7B+6N8WfQMtDLX2Dpw==} engines: {node: '>= 18'} + '@octokit/request@9.1.3': + resolution: {integrity: sha512-V+TFhu5fdF3K58rs1pGUJIDH5RZLbZm5BI+MNF+6o/ssFNT4vWlCh/tVpF3NxGtP15HUxTTMUbsG5llAuU2CZA==} + engines: {node: '>= 18'} + '@octokit/types@12.6.0': resolution: {integrity: sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==} '@octokit/types@13.5.0': resolution: {integrity: sha512-HdqWTf5Z3qwDVlzCrP8UJquMwunpDiMPt5er+QjGzL4hqr/vBVY/MauQgS1xWxCDT1oMx1EULyqxncdCY/NVSQ==} + '@octokit/types@13.6.2': + resolution: {integrity: sha512-WpbZfZUcZU77DrSW4wbsSgTPfKcp286q3ItaIgvSbBpZJlu6mnYXAkjZz6LVZPXkEvLIM8McanyZejKTYUHipA==} + '@octokit/webhooks-methods@4.1.0': resolution: {integrity: sha512-zoQyKw8h9STNPqtm28UGOYFE7O6D4Il8VJwhAtMHFt2C4L0VQT1qGKLeefUOqHNs1mNRYSadVv7x0z8U2yyeWQ==} engines: {node: '>= 18'} @@ -643,6 +691,9 @@ packages: before-after-hook@2.2.3: resolution: {integrity: sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==} + before-after-hook@3.0.2: + resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==} + binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} @@ -2449,6 +2500,9 @@ packages: universal-user-agent@6.0.1: resolution: {integrity: sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==} + universal-user-agent@7.0.2: + resolution: {integrity: sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==} + update-browserslist-db@1.1.0: resolution: {integrity: sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==} hasBin: true @@ -2716,6 +2770,8 @@ snapshots: '@octokit/auth-token@4.0.0': {} + '@octokit/auth-token@5.1.1': {} + '@octokit/auth-unauthenticated@5.0.1': dependencies: '@octokit/request-error': 5.1.0 @@ -2731,6 +2787,21 @@ snapshots: before-after-hook: 2.2.3 universal-user-agent: 6.0.1 + '@octokit/core@6.1.2': + dependencies: + '@octokit/auth-token': 5.1.1 + '@octokit/graphql': 8.1.1 + '@octokit/request': 9.1.3 + '@octokit/request-error': 6.1.5 + '@octokit/types': 13.5.0 + before-after-hook: 3.0.2 + universal-user-agent: 7.0.2 + + '@octokit/endpoint@10.1.1': + dependencies: + '@octokit/types': 13.5.0 + universal-user-agent: 7.0.2 + '@octokit/endpoint@9.0.5': dependencies: '@octokit/types': 13.5.0 @@ -2742,6 +2813,12 @@ snapshots: '@octokit/types': 13.5.0 universal-user-agent: 6.0.1 + '@octokit/graphql@8.1.1': + dependencies: + '@octokit/request': 9.1.3 + '@octokit/types': 13.5.0 + universal-user-agent: 7.0.2 + '@octokit/oauth-app@6.1.0': dependencies: '@octokit/auth-oauth-app': 7.1.0 @@ -2771,11 +2848,20 @@ snapshots: dependencies: '@octokit/core': 5.2.0 + '@octokit/plugin-paginate-graphql@5.2.4(@octokit/core@6.1.2)': + dependencies: + '@octokit/core': 6.1.2 + '@octokit/plugin-paginate-rest@11.3.1(@octokit/core@5.2.0)': dependencies: '@octokit/core': 5.2.0 '@octokit/types': 13.5.0 + '@octokit/plugin-paginate-rest@11.3.6(@octokit/core@6.1.2)': + dependencies: + '@octokit/core': 6.1.2 + '@octokit/types': 13.6.2 + '@octokit/plugin-paginate-rest@9.2.1(@octokit/core@5.2.0)': dependencies: '@octokit/core': 5.2.0 @@ -2805,6 +2891,10 @@ snapshots: deprecation: 2.3.1 once: 1.4.0 + '@octokit/request-error@6.1.5': + dependencies: + '@octokit/types': 13.5.0 + '@octokit/request@8.4.0': dependencies: '@octokit/endpoint': 9.0.5 @@ -2812,6 +2902,13 @@ snapshots: '@octokit/types': 13.5.0 universal-user-agent: 6.0.1 + '@octokit/request@9.1.3': + dependencies: + '@octokit/endpoint': 10.1.1 + '@octokit/request-error': 6.1.5 + '@octokit/types': 13.5.0 + universal-user-agent: 7.0.2 + '@octokit/types@12.6.0': dependencies: '@octokit/openapi-types': 20.0.0 @@ -2820,6 +2917,10 @@ snapshots: dependencies: '@octokit/openapi-types': 22.2.0 + '@octokit/types@13.6.2': + dependencies: + '@octokit/openapi-types': 22.2.0 + '@octokit/webhooks-methods@4.1.0': {} '@octokit/webhooks-types@7.4.0': {} @@ -3111,6 +3212,8 @@ snapshots: before-after-hook@2.2.3: {} + before-after-hook@3.0.2: {} + binary-extensions@2.3.0: {} bottleneck@2.19.5: {} @@ -5305,6 +5408,8 @@ snapshots: universal-user-agent@6.0.1: {} + universal-user-agent@7.0.2: {} + update-browserslist-db@1.1.0(browserslist@4.23.2): dependencies: browserslist: 4.23.2 diff --git a/scraper/package.json b/scraper/package.json index af61d553..5136b536 100644 --- a/scraper/package.json +++ b/scraper/package.json @@ -6,13 +6,16 @@ "type": "module", "scripts": { "build": "tsc", - "start": "node dist/index.js", + "start": "node dist/github-scraper/index.js", "dev": "pnpm build && pnpm start" }, "keywords": [], "author": "", "license": "ISC", "dependencies": { + "@octokit/core": "^6.1.2", + "@octokit/plugin-paginate-graphql": "^5.2.4", + "@octokit/plugin-paginate-rest": "^11.3.6", "date-fns": "^3.6.0", "dotenv": "^16.4.5", "octokit": "^4.0.2" diff --git a/scraper/pnpm-lock.yaml b/scraper/pnpm-lock.yaml index 8abc1bae..29473165 100644 --- a/scraper/pnpm-lock.yaml +++ b/scraper/pnpm-lock.yaml @@ -8,6 +8,15 @@ importers: .: dependencies: + '@octokit/core': + specifier: ^6.1.2 + version: 6.1.2 + '@octokit/plugin-paginate-graphql': + specifier: ^5.2.4 + version: 5.2.4(@octokit/core@6.1.2) + '@octokit/plugin-paginate-rest': + specifier: ^11.3.6 + version: 11.3.6(@octokit/core@6.1.2) date-fns: specifier: ^3.6.0 version: 3.6.0 @@ -85,14 +94,14 @@ packages: '@octokit/openapi-webhooks-types@8.3.0': resolution: {integrity: sha512-vKLsoR4xQxg4Z+6rU/F65ItTUz/EXbD+j/d4mlq2GW8TsA4Tc8Kdma2JTAAJ5hrKWUQzkR/Esn2fjsqiVRYaQg==} - '@octokit/plugin-paginate-graphql@5.2.2': - resolution: {integrity: sha512-7znSVvlNAOJisCqAnjN1FtEziweOHSjPGAuc5W58NeGNAr/ZB57yCsjQbXDlWsVryA7hHQaEQPcBbJYFawlkyg==} + '@octokit/plugin-paginate-graphql@5.2.4': + resolution: {integrity: sha512-pLZES1jWaOynXKHOqdnwZ5ULeVR6tVVCMm+AUbp0htdcyXDU95WbkYdU4R2ej1wKj5Tu94Mee2Ne0PjPO9cCyA==} engines: {node: '>= 18'} peerDependencies: '@octokit/core': '>=6' - '@octokit/plugin-paginate-rest@11.3.3': - resolution: {integrity: sha512-o4WRoOJZlKqEEgj+i9CpcmnByvtzoUYC6I8PD2SA95M+BJ2x8h7oLcVOg9qcowWXBOdcTRsMZiwvM3EyLm9AfA==} + '@octokit/plugin-paginate-rest@11.3.6': + resolution: {integrity: sha512-zcvqqf/+TicbTCa/Z+3w4eBJcAxCFymtc0UAIsR3dEVoNilWld4oXdscQ3laXamTszUZdusw97K8+DrbFiOwjw==} engines: {node: '>= 18'} peerDependencies: '@octokit/core': '>=6' @@ -126,6 +135,9 @@ packages: '@octokit/types@13.5.0': resolution: {integrity: sha512-HdqWTf5Z3qwDVlzCrP8UJquMwunpDiMPt5er+QjGzL4hqr/vBVY/MauQgS1xWxCDT1oMx1EULyqxncdCY/NVSQ==} + '@octokit/types@13.6.2': + resolution: {integrity: sha512-WpbZfZUcZU77DrSW4wbsSgTPfKcp286q3ItaIgvSbBpZJlu6mnYXAkjZz6LVZPXkEvLIM8McanyZejKTYUHipA==} + '@octokit/webhooks-methods@5.1.0': resolution: {integrity: sha512-yFZa3UH11VIxYnnoOYCVoJ3q4ChuSOk2IVBBQ0O3xtKX4x9bmKb/1t+Mxixv2iUhzMdOl1qeWJqEhouXXzB3rQ==} engines: {node: '>= 18'} @@ -182,7 +194,7 @@ snapshots: '@octokit/auth-unauthenticated': 6.1.0 '@octokit/core': 6.1.2 '@octokit/oauth-app': 7.1.3 - '@octokit/plugin-paginate-rest': 11.3.3(@octokit/core@6.1.2) + '@octokit/plugin-paginate-rest': 11.3.6(@octokit/core@6.1.2) '@octokit/types': 13.5.0 '@octokit/webhooks': 13.3.0 @@ -233,7 +245,7 @@ snapshots: '@octokit/graphql': 8.1.1 '@octokit/request': 9.1.3 '@octokit/request-error': 6.1.4 - '@octokit/types': 13.5.0 + '@octokit/types': 13.6.2 before-after-hook: 3.0.2 universal-user-agent: 7.0.2 @@ -245,7 +257,7 @@ snapshots: '@octokit/graphql@8.1.1': dependencies: '@octokit/request': 9.1.3 - '@octokit/types': 13.5.0 + '@octokit/types': 13.6.2 universal-user-agent: 7.0.2 '@octokit/oauth-app@7.1.3': @@ -272,14 +284,14 @@ snapshots: '@octokit/openapi-webhooks-types@8.3.0': {} - '@octokit/plugin-paginate-graphql@5.2.2(@octokit/core@6.1.2)': + '@octokit/plugin-paginate-graphql@5.2.4(@octokit/core@6.1.2)': dependencies: '@octokit/core': 6.1.2 - '@octokit/plugin-paginate-rest@11.3.3(@octokit/core@6.1.2)': + '@octokit/plugin-paginate-rest@11.3.6(@octokit/core@6.1.2)': dependencies: '@octokit/core': 6.1.2 - '@octokit/types': 13.5.0 + '@octokit/types': 13.6.2 '@octokit/plugin-rest-endpoint-methods@13.2.4(@octokit/core@6.1.2)': dependencies: @@ -314,6 +326,10 @@ snapshots: dependencies: '@octokit/openapi-types': 22.2.0 + '@octokit/types@13.6.2': + dependencies: + '@octokit/openapi-types': 22.2.0 + '@octokit/webhooks-methods@5.1.0': {} '@octokit/webhooks@13.3.0': @@ -343,8 +359,8 @@ snapshots: '@octokit/app': 15.1.0 '@octokit/core': 6.1.2 '@octokit/oauth-app': 7.1.3 - '@octokit/plugin-paginate-graphql': 5.2.2(@octokit/core@6.1.2) - '@octokit/plugin-paginate-rest': 11.3.3(@octokit/core@6.1.2) + '@octokit/plugin-paginate-graphql': 5.2.4(@octokit/core@6.1.2) + '@octokit/plugin-paginate-rest': 11.3.6(@octokit/core@6.1.2) '@octokit/plugin-rest-endpoint-methods': 13.2.4(@octokit/core@6.1.2) '@octokit/plugin-retry': 7.1.1(@octokit/core@6.1.2) '@octokit/plugin-throttling': 9.3.1(@octokit/core@6.1.2) diff --git a/scraper/src/github-scraper/config.ts b/scraper/src/github-scraper/config.ts index 18389861..0a1ffbd5 100644 --- a/scraper/src/github-scraper/config.ts +++ b/scraper/src/github-scraper/config.ts @@ -1,4 +1,8 @@ -import { Octokit } from "octokit"; +import { Octokit } from "@octokit/core"; +import { paginateRest } from "@octokit/plugin-paginate-rest"; +import { paginateGraphQL } from "@octokit/plugin-paginate-graphql"; + +const MyOctokit = Octokit.plugin(paginateRest, paginateGraphQL); const GITHUB_TOKEN = process.env.GITHUB_TOKEN; @@ -7,6 +11,6 @@ if (!GITHUB_TOKEN) { process.exit(1); } -export const octokit = new Octokit({ +export const octokit = new MyOctokit({ auth: GITHUB_TOKEN, }); diff --git a/scraper/src/github-scraper/fetchEvents.ts b/scraper/src/github-scraper/fetchEvents.ts index 50dce7cd..1913de32 100644 --- a/scraper/src/github-scraper/fetchEvents.ts +++ b/scraper/src/github-scraper/fetchEvents.ts @@ -1,50 +1,76 @@ import { octokit } from "./config.js"; import dotenv from "dotenv"; + dotenv.config(); -const blacklistedUsers = [ +const BlacklistedUsers = [ "dependabot", "snyk-bot", "codecov-commenter", "github-actions[bot]", ].concat(process.env.BLACKLISTED_USERS?.split(",") ?? []); -const requiredEventType = [ +const AllowedEventTypes = [ "IssueCommentEvent", "IssuesEvent", "PullRequestEvent", "PullRequestReviewEvent", ]; -export const fetchEvents = async ( - org: string, - startDate: Date, - endDate: Date, -) => { - const events = await octokit.paginate("GET /orgs/{org}/events", { - org: org, - per_page: 1000, - }); - - const filteredEvents = []; - for (const event of events) { - const eventTime: Date = new Date(event.created_at ?? 0); - - if (eventTime > endDate) { - continue; - } else if (eventTime <= startDate) { - return filteredEvents; - } - - if ( - !blacklistedUsers.includes(event.actor.login) && - event.type && - requiredEventType.includes(event.type) - ) { - filteredEvents.push(event); - } - } - console.log("Fetched " + filteredEvents.length + " events"); - - return filteredEvents; +export const fetchEvents = async (org: string) => { + const events = await octokit.paginate( + "GET /orgs/{org}/events", + { org, per_page: 100 }, + (response) => { + const filtered = response.data.filter((event) => { + if (!event.type) { + console.debug(`Skipping event without type`); + return false; + } + if (event.actor.login.includes("[bot]")) { + return false; + } + if (BlacklistedUsers.includes(event.actor.login)) { + return false; + } + if (!AllowedEventTypes.includes(event.type)) { + return false; + } + return true; + }); + console.log( + `Pulled ${filtered.length} relevant events out of ${response.data.length} events from page ${response.url}`, + ); + return filtered; + }, + ); + + console.log( + `Pulled ${events.length} events in total ranging from ${new Date( + events[0].created_at ?? "", + ).toLocaleString()} to ${new Date( + events[events.length - 1].created_at ?? "", + ).toLocaleString()} (${getDurationString( + new Date(events[0].created_at ?? ""), + new Date(events[events.length - 1].created_at ?? ""), + )})`, + ); + + console.log( + `The latest event is ${getDurationString( + new Date(), + new Date(events[0].created_at ?? ""), + )} behind from now`, + ); + + return events; +}; + +const getDurationString = (a: Date, b: Date) => { + const diff = Math.abs(b.getTime() - a.getTime()); + const hours = Math.floor(diff / 1000 / 60 / 60); + const minutes = Math.floor((diff / 1000 / 60) % 60); + return `${hours.toString().padStart(2, "0")}h ${minutes + .toString() + .padStart(2, "0")}m`; }; diff --git a/scraper/src/github-scraper/index.ts b/scraper/src/github-scraper/index.ts index e8ae8274..6b331a02 100644 --- a/scraper/src/github-scraper/index.ts +++ b/scraper/src/github-scraper/index.ts @@ -1,23 +1,17 @@ import { formatISO, parseISO, subDays } from "date-fns"; -import { fetchMergeEvents, fetchOpenPulls } from "./fetchUserData.js"; import { IGitHubEvent, ProcessData } from "./types.js"; import { fetchEvents } from "./fetchEvents.js"; -import { parseEvents } from "./parseEvents.js"; import { mergedData } from "./saveData.js"; import { scrapeDiscussions } from "./discussion.js"; import scrapeProjectBoardItems from "./projectItems.js"; +import { parseEvents } from "./parseEvents.js"; +import { fetchMergeEvents, fetchOpenPulls } from "./fetchUserData.js"; let processedData: ProcessData = {}; -const scrapeGitHub = async ( - org: string, - endDate: Date, - startDate: Date, -): Promise => { - console.log( - `Scraping GitHub data for ${org} from ${formatISO(startDate)} to ${formatISO(endDate)}`, - ); - const events = await fetchEvents(org, startDate, endDate); +const scrapeGitHub = async (org: string): Promise => { + console.log(`Scraping GitHub data for: '${org}'`); + const events = await fetchEvents(org); processedData = await parseEvents(events as IGitHubEvent[]); for (const user of Object.keys(processedData)) { if (!processedData[user]) { @@ -75,7 +69,7 @@ const main = async () => { const endDate = parseISO(date); const startDate = subDays(endDate, Number(numDays)); - await scrapeGitHub(orgName, endDate, startDate); + await scrapeGitHub(orgName); await mergedData(dataDir, processedData); await scrapeDiscussions(orgName, dataDir, endDate, startDate); diff --git a/scraper/src/github-scraper/parseEvents.ts b/scraper/src/github-scraper/parseEvents.ts index dd62526a..f3a1650c 100644 --- a/scraper/src/github-scraper/parseEvents.ts +++ b/scraper/src/github-scraper/parseEvents.ts @@ -27,7 +27,9 @@ async function getDefaultBranch(owner: string, repo: string) { return defaultBranches[repo]; } function appendEvent(user: string, event: Activity) { - console.log(`Appending event for ${user}`); + console.debug( + `Appending ${event.type} event for user ${user}. ${event.link}`, + ); if (!processedData[user]) { console.log(`Creating new user data for ${user}`); processedData[user] = { diff --git a/scraper/tsconfig.json b/scraper/tsconfig.json index 6855b5dc..5bd2aad3 100644 --- a/scraper/tsconfig.json +++ b/scraper/tsconfig.json @@ -1,15 +1,19 @@ { "compilerOptions": { - "target": "ESNext", + "target": "ES2020", "module": "ESNext", "moduleResolution": "node", - "outDir": "./dist", "esModuleInterop": true, - "downlevelIteration": true, - "forceConsistentCasingInFileNames": true, "strict": true, - "skipLibCheck": true + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "resolveJsonModule": true, + "types": ["node"], + "sourceMap": true }, - "include": ["src/github-scraper/**/*", "dist/discssion.js"], - "exclude": ["node_modules"] + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] } From 675f2e5b07bf58ea99d20f13b43062f992d76394 Mon Sep 17 00:00:00 2001 From: rithviknishad Date: Thu, 28 Nov 2024 22:06:45 +0530 Subject: [PATCH 4/5] Show when page was built in footer --- components/Footer.tsx | 93 ++++++++++++++++++++---------------- components/navbar/Navbar.tsx | 2 +- 2 files changed, 53 insertions(+), 42 deletions(-) diff --git a/components/Footer.tsx b/components/Footer.tsx index d86021bf..3afdce88 100644 --- a/components/Footer.tsx +++ b/components/Footer.tsx @@ -3,6 +3,7 @@ import { env } from "@/env.mjs"; import { FaYoutube, FaLinkedin, FaGithub, FaEnvelope } from "react-icons/fa"; import { ReactNode } from "react"; import { BsTwitterX } from "react-icons/bs"; +import RelativeTime from "@/components/RelativeTime"; const SocialLink = ({ href, @@ -84,48 +85,58 @@ export default function Footer() {
-
- - - Data Repository - - - Flat Repository Explorer - - +
+
+ + + Data Repository + + + Flat Repository Explorer + + - -
- - - - - -
- + +
+ + + + + +
+ +
+ + +

Leaderboard scrapes data frequently.

+

+ Data was last updated{" "} + . +

+
diff --git a/components/navbar/Navbar.tsx b/components/navbar/Navbar.tsx index b15a17b1..93a6105c 100644 --- a/components/navbar/Navbar.tsx +++ b/components/navbar/Navbar.tsx @@ -28,7 +28,7 @@ export default function Navbar() { const pathname = usePathname(); return ( <> -