-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
783 backend write a cron job to automatically update a google sheet w…
…ith the members (#791) * Create cron job to auto update google sheet with members Draft PR. * Created new cron job(note that the cron is currently disabled and has workflow-dispatch for now). * Created `update-google-sheet-members.ts` script that current just fetches all users. * Add basic error handling to fetching all members * Add googleapis to package.json * Add yarn lock for googleapis * Finalise logic for update google sheet members script * Create `fetchUsers` method that fetches users from the backend * Create `getAllUsers` that fetches all the users from the backend using `fetchUsers` * Create `authenticateGoogle` method that returns a google auth client * Note that documentation will come soon... * Create `appendToGoogleSheet` and `clearSheet` * `appendToGoogleSheet` takes in the logs and auth client and updates the google sheet rows * `clearSheet` clears the google sheet so that `appendToGoogleSheet` doesn't overlap everything Note that the logic is still somewhat rough and could have improvements. Quite a lot of duplicate code and possibly better ways to update the google sheet instead of just appending and clearing every time. * Add googleapis to dev dependencies * Update script * Now creates admin jwt with a function internally instead of with the actions script. * Moved all logic into an asynchronous function * Update date time formatting to a more readable format. * Update google sheet action and add env variables required * Update secret used in scipt and use VITE_FIREBASE_API_KEY instead of API_KEY * Update environment variables and show on top of file * Update the environment variable to SPREADSHEET_ID for clarity Used SHEET_ID before and apparently thats a different thing * Update environment variables once again * Uses NEXT instead of old VITE but references to VITE due to secrets not updated in github. * Updated env to include both `SHEET_ID` and `SPREADSHEET_ID`. * Update to filter only members * Remove unecessary comments * Update env variable naming and add docs for createIdToken * Update script to not use env user_id but hard coded user_id * Not add email to user creation during jwt generation
- Loading branch information
1 parent
eb66544
commit fc82826
Showing
4 changed files
with
341 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| name: Update the google sheet with members | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| schedule: | ||
| # Updates every 00:00 every day | ||
| - cron: "0 0 * * *" | ||
|
|
||
| env: | ||
| NEXT_PUBLIC_BACKEND_BASE_URL: ${{secrets.VITE_BACKEND_BASE_URL}} | ||
| MEMBERS_GOOGLE_SPREADSHEET_ID: ${{secrets.MEMBERS_GOOGLE_SPREADSHEET_ID}} | ||
| MEMBERS_GOOGLE_SHEET_ID: ${{secrets.MEMBERS_GOOGLE_SHEET_ID}} | ||
| GOOGLE_SERVICE_ACCOUNT_JSON: ${{secrets.GOOGLE_SERVICE_ACCOUNT_JSON}} | ||
| NEXT_PUBLIC_FIREBASE_API_KEY: ${{secrets.VITE_FIREBASE_API_KEY}} | ||
|
|
||
| jobs: | ||
| update_google_sheet_members: | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 10 | ||
|
|
||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| ref: stable | ||
|
|
||
| - name: Install volta | ||
| uses: volta-cli/action@v4 | ||
|
|
||
| - name: Update google sheet members with script | ||
| run: | | ||
| yarn workspace server update-google-sheet-members |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,239 @@ | ||
| import { CombinedUserData } from "service-layer/response-models/UserResponse" | ||
| import { firestoreTimestampToDate } from "../src/data-layer/adapters/DateUtils" | ||
|
|
||
| import { google } from "googleapis" | ||
| import admin from "firebase-admin" | ||
| import dotenv from "dotenv" | ||
|
|
||
| dotenv.config() | ||
|
|
||
| // Environment variables | ||
| const BASE_URL = process.env.NEXT_PUBLIC_BACKEND_BASE_URL | ||
| const MEMBERS_GOOGLE_SPREADSHEET_ID = process.env.MEMBERS_GOOGLE_SPREADSHEET_ID | ||
| const MEMBERS_GOOGLE_SHEET_ID = process.env.MEMBERS_GOOGLE_SHEET_ID | ||
| const API_KEY = process.env.NEXT_PUBLIC_FIREBASE_API_KEY | ||
| const GOOGLE_SERVICE_ACCOUNT_JSON = process.env.GOOGLE_SERVICE_ACCOUNT_JSON | ||
| const USER_ID = "google-sheets-bot" | ||
|
|
||
| admin.initializeApp({ | ||
| credential: admin.credential.cert(JSON.parse(GOOGLE_SERVICE_ACCOUNT_JSON)) | ||
| }) | ||
|
|
||
| const categories = [ | ||
| "Email", | ||
| "First name", | ||
| "Last name", | ||
| "Phone number", | ||
| "Date of birth", | ||
| "Membership", | ||
| "Date joined", | ||
| "University year", | ||
| "Gender", | ||
| "Does racing", | ||
| "Does ski", | ||
| "Does snowboarding", | ||
| "Emergency contact", | ||
| "Dietary requirements", | ||
| "UID" | ||
| ] | ||
|
|
||
| /** | ||
| * Fetches users from the backend | ||
| * @param token - The token to authenticate the request | ||
| * @param cursor - The cursor to fetch the next page of users | ||
| * @returns The data json object from the response | ||
| */ | ||
| async function fetchUsers(token: string, cursor?: string): Promise<any> { | ||
| const res = await fetch( | ||
| // Note that VITE_BACKEND_BASE_URL does have a slash at the end | ||
| `${BASE_URL}admin/users${cursor ? `?cursor=${cursor}` : ""}`, | ||
| { | ||
| method: "GET", | ||
| headers: { | ||
| accept: "application/json", | ||
| authorization: `Bearer ${token}` | ||
| } | ||
| } | ||
| ) | ||
| if (res.status === 200) { | ||
| const data = await res.json() | ||
| return data | ||
| } else { | ||
| throw new Error(`Failed to fetch users, status: ${res.status}`) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Fetches all users from the backend | ||
| * @param token - The token to authenticate the request | ||
| * @returns An array of CombinedUserData | ||
| */ | ||
| async function getAllUsers(token: string): Promise<CombinedUserData[]> { | ||
| const allUsers: CombinedUserData[] = [] | ||
| let fetchedUsers = await fetchUsers(token) | ||
| allUsers.push(...fetchedUsers.data) | ||
| while (fetchedUsers.cursor) { | ||
| fetchedUsers = await fetchUsers(token, fetchedUsers.cursor) | ||
| allUsers.push(...fetchedUsers.data) | ||
| } | ||
| return allUsers | ||
| } | ||
|
|
||
| /** | ||
| * Creates google authentication client | ||
| * @returns The google auth client | ||
| */ | ||
| async function authenticateGoogle(): Promise<any> { | ||
| const { client_email, private_key } = JSON.parse(GOOGLE_SERVICE_ACCOUNT_JSON) | ||
| const auth = new google.auth.GoogleAuth({ | ||
| credentials: { | ||
| client_email, | ||
| private_key | ||
| }, | ||
| scopes: ["https://www.googleapis.com/auth/spreadsheets"] | ||
| }) | ||
|
|
||
| const authClient = await auth.getClient() | ||
| return authClient | ||
| } | ||
|
|
||
| /** | ||
| * Append data to Google Sheets using API Key | ||
| * @param auth - The google auth client | ||
| * @param rows - The rows to append to the Google Sheet | ||
| * @returns The response from the Google Sheets API | ||
| */ | ||
| async function updateGoogleSheet(auth: any, rows: any[]) { | ||
| const sheets = google.sheets({ | ||
| version: "v4", | ||
| auth | ||
| }) | ||
|
|
||
| const request = { | ||
| spreadsheetId: MEMBERS_GOOGLE_SPREADSHEET_ID, | ||
| // Sheet id is something like "Sheet1" | ||
| range: MEMBERS_GOOGLE_SHEET_ID + "!A1", | ||
| valueInputOption: "RAW", | ||
| insertDataOption: "INSERT_ROWS", | ||
| resource: { | ||
| values: rows | ||
| } | ||
| } | ||
|
|
||
| try { | ||
| const response = (await sheets.spreadsheets.values.append(request)).data | ||
| console.log(`${rows.length} rows added to the google sheet.`) | ||
| return response | ||
| } catch (err) { | ||
| return console.error("Failed to add rows to the google sheet.", err) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Clears google sheet data. | ||
| * @param auth - The google auth client | ||
| */ | ||
| async function clearSheet(auth: any) { | ||
| const sheets = google.sheets({ | ||
| version: "v4", | ||
| auth | ||
| }) | ||
|
|
||
| const request = { | ||
| spreadsheetId: MEMBERS_GOOGLE_SPREADSHEET_ID, | ||
| range: MEMBERS_GOOGLE_SHEET_ID | ||
| } | ||
| try { | ||
| await sheets.spreadsheets.values.clear(request) | ||
| console.log(`Cleared google sheet.`) | ||
| } catch (err) { | ||
| console.error("Failed to clear the Google Sheet", err) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Converts user information to an array that google sheets can parse. | ||
| * @param users An array of all CombinedUserData | ||
| * @returns The mapper user arrays | ||
| */ | ||
| function mapUsers(users: CombinedUserData[]) { | ||
| return users.map((user: CombinedUserData) => [ | ||
| user.email, | ||
| user.first_name, | ||
| user.last_name, | ||
| user.phone_number, | ||
| firestoreTimestampToDate(user.date_of_birth).toLocaleString([], { | ||
| timeZone: "Pacific/Auckland", | ||
| hour12: true | ||
| }), | ||
| user.membership, | ||
| user.dateJoined, | ||
| user.university_year, | ||
| user.gender, | ||
| user.does_racing, | ||
| user.does_ski, | ||
| user.does_snowboarding, | ||
| user.emergency_contact, | ||
| // Remove stripe id from fields | ||
| user.dietary_requirements, | ||
| user.uid | ||
| ]) | ||
| } | ||
|
|
||
| /** | ||
| * Code from login-prod.ts to create admin jwt token | ||
| * @param uid - The user id to create the token for | ||
| * @returns The jwt token | ||
| */ | ||
| const createIdToken = async () => { | ||
| try { | ||
| // Ensure that the user exists | ||
| try { | ||
| await admin.auth().getUser(USER_ID) | ||
| } catch (e) { | ||
| await admin.auth().createUser({ uid: USER_ID }) | ||
| } | ||
| await admin | ||
| .auth() | ||
| .setCustomUserClaims(USER_ID, { member: true, admin: true }) | ||
|
|
||
| const customToken = await admin.auth().createCustomToken(USER_ID) | ||
| const res = await fetch( | ||
| `https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyCustomToken?key=${API_KEY}`, | ||
| { | ||
| method: "POST", | ||
| headers: { | ||
| Accept: "application/json", | ||
| "Content-Type": "application/json" | ||
| }, | ||
| body: JSON.stringify({ | ||
| token: customToken, | ||
| returnSecureToken: true | ||
| }) | ||
| } | ||
| ) | ||
|
|
||
| const data = (await res.json()) as any | ||
|
|
||
| return data.idToken | ||
| } catch (e) { | ||
| console.error(e) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Updates the google sheet with members fetched from backend | ||
| * @param token - The token to authenticate the request | ||
| */ | ||
| async function updateGoogleSheetMembers() { | ||
| const token = await createIdToken() | ||
| const allUsers: CombinedUserData[] = (await getAllUsers(token)).filter( | ||
| (user) => user.membership === "member" | ||
| ) | ||
| const rows = mapUsers(allUsers) | ||
| const auth = await authenticateGoogle() | ||
| await clearSheet(auth) // Clear sheet first | ||
| await updateGoogleSheet(auth, [categories, ...rows]) | ||
| } | ||
|
|
||
| updateGoogleSheetMembers() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters