web: Implement AI reports

web/api/context.ts

@@ -1,8 +1,8 @@
 import { inferAsyncReturnType } from '@trpc/server'
-import { CreateNextContextOptions } from '@trpc/server/adapters/next'
+import { CreateExpressContextOptions } from '@trpc/server/adapters/express'
 import { isbot } from 'isbot'
 
-export async function createContext(opts: CreateNextContextOptions) {
+export async function createContext(opts: CreateExpressContextOptions) {
   const isBot = isbot(opts.req.headers['user-agent'] || '')
 
   return {

web/api/index.ts

@@ -9,25 +9,58 @@ import { router } from './trpc'
 import renderRouter from './routes/render'
 import klickerRouter from './routes/klicker'
 import etag from 'etag'
+import { reportRouter, updateAllReports, updateReport } from './routes/report'
 
 const appRouter = router({
   player: playerRouter,
   club: clubRouter,
   rankings: rankingsRouter,
   events: eventsRouter,
+  report: reportRouter,
 })
 
 export type AppRouter = typeof appRouter
 
 const app = express()
 
 // TODO restrict to localhost
-app.post('/cron', async (req, res) => {
+app.post('/cron', async (req, res, next) => {
   console.time('running cron jobs')
-  const profileUpdater = await updateAllProfiles()
+  try {
+    const summary = await updateAllProfiles()
+    res.json(summary)
+  } catch (err) {
+    console.error(err)
+    next(err)
+  }
   console.timeEnd('running cron jobs')
+})
+
+// TODO move to cron
+app.post('/update-reports', async (req, res, next) => {
+  console.time('reports update')
+  try {
+    const summary = await updateAllReports()
+    res.json(summary)
+  } catch (err) {
+    console.error(err)
+    next(err)
+  }
+  console.timeEnd('reports update')
+})
 
-  res.json({ profileUpdater })
+// TODO remove, only for testing
+app.post('/update-report', express.json(), async (req, res, next) => {
+  console.time('report update')
+  try {
+    const { locale, mode, map } = req.body
+    const summary = await updateReport(locale as string, mode as string, map as string)
+    res.json(summary)
+  } catch (err) {
+    console.error(err)
+    next(err)
+  }
+  console.timeEnd('report update')
 })
 
 app.use('/render', renderRouter)
@@ -51,7 +84,7 @@ app.use(
         return {
           headers: {
             etag: etag(JSON.stringify(data)),
-            'cache-control': ctx.res.getHeader('cache-control') ?? 'public, max-age=0, stale-if-error=86400',
+            'cache-control': ctx.res.getHeader('cache-control') as string ?? 'public, max-age=0, stale-if-error=86400',
           },
         };
       }

web/api/migrations/20240208201900_create_map_report.ts

@@ -0,0 +1,18 @@
+import { Knex } from 'knex'
+
+export async function up(knex: Knex): Promise<void> {
+  return knex.schema.createTable('map_report', (table) => {
+    table.increments('id').primary()
+    table.string('locale_iso').notNullable()
+    table.string('mode').notNullable()
+    table.string('map').notNullable()
+    table.datetime('timestamp').notNullable()
+    table.text('prompt').notNullable()
+    table.text('completion').notNullable()
+    table.index(['locale_iso', 'mode', 'map'])
+  })
+}
+
+export async function down(knex: Knex): Promise<void> {
+  return knex.schema.dropTable('map_report')
+}

web/api/routes/render.ts

@@ -26,7 +26,9 @@ function asyncWrapper(fn: RequestHandler) {
 
 const profileView = new ProfileView();
 const brawlStarsApiService = new BrawlstarsService();
-const klickerService = new BrawltimeKlickerService(process.env.CUBE_URL!, fetch);
+const CUBE_URL = process.env.CUBE_URL
+
+const klickerService = new BrawltimeKlickerService(CUBE_URL, fetch);
 
 async function getPlayerTotals(tag: string) {
   return await klickerService.query({

web/api/routes/report.ts

@@ -0,0 +1,118 @@
+import Knex from 'knex'
+import MarkdownIt from 'markdown-it'
+import knexfile from '../knexfile'
+import { fetch, Agent } from 'undici'
+import ReportGeneratorService from '../services/ReportGeneratorService'
+import { BrawltimeKlickerService } from '~/plugins/klicker.service'
+import { publicProcedure, router } from '../trpc'
+import * as z from 'zod'
+import { TRPCError } from '@trpc/server'
+import { locales } from '~/locales'
+
+let reportGeneratorService: ReportGeneratorService | undefined
+
+const CUBE_URL = process.env.CUBE_URL
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY
+
+if (process.env.MYSQL_HOST && CUBE_URL != undefined && OPENAI_API_KEY != undefined) {
+  const environment = process.env.NODE_ENV || 'development'
+  const knex = Knex(knexfile[environment])
+
+  const agent = new Agent({
+    bodyTimeout: 30000,
+    headersTimeout: 30000,
+  })
+
+  const klickerService = new BrawltimeKlickerService(CUBE_URL, (url, input) => fetch(url as string, {
+    ...input as any,
+    dispatcher: agent,
+  }) as any)
+
+  reportGeneratorService = new ReportGeneratorService(OPENAI_API_KEY, knex, klickerService)
+} else {
+  if (CUBE_URL == undefined) {
+    console.warn('CUBE_URL is not set, report generator will be unavailable')
+  }
+  if (OPENAI_API_KEY == undefined) {
+    console.warn('OPENAI_API_KEY is not set, report generator will be unavailable')
+  }
+  if (process.env.MYSQL_HOST == undefined) {
+    console.warn('MYSQL_HOST is not set, report generator will be unavailable')
+  }
+}
+
+export async function updateAllReports() {
+  if (reportGeneratorService == undefined) {
+    console.warn('Report generator is not available')
+    return { error: 'Report generator is not available' }
+  }
+
+  const activeEvents = await reportGeneratorService.getActiveEvents()
+  console.log('Updating reports for active events')
+
+  for (const activeEvent of activeEvents) {
+    // TODO non-latin languages need more output tokens, exceeding the 4k output limit - chunk the prompt for them
+    const latinLocales = ['de', 'en', 'it', 'pl', 'es'] // top 5 locales for the test phase
+    for (const locale of locales.filter(l => latinLocales.includes(l.iso))) {
+      await reportGeneratorService.updateReportsForMap(locale, activeEvent)
+      console.log('Updated report for', locale.iso, activeEvent)
+    }
+  }
+
+  return { events: activeEvents.length }
+}
+
+export async function updateReport(localeIso: string, mode: string, map: string) {
+  if (reportGeneratorService == undefined) {
+    console.warn('Report generator is not available')
+    return { error: 'Report generator is not available' }
+  }
+
+  const locale = locales.find(l => l.iso == localeIso)
+
+  if (locale == undefined) {
+    return { error: 'Unknown locale' }
+  }
+
+  console.log('Updating reports for', { localeIso, mode, map })
+
+  await reportGeneratorService.updateReportsForMap(locale, { mode, map })
+  console.log('Updated report for', { localeIso, mode, map })
+
+  return { events: 1 }
+}
+
+export const reportRouter = router({
+  byModeMap: publicProcedure
+    .input(z.object({
+      mode: z.string(),
+      map: z.string(),
+      localeIso: z.string(),
+    }))
+    .query(async ({ input, ctx }) => {
+      if (reportGeneratorService == undefined) {
+        throw new TRPCError({
+          code: 'NOT_FOUND',
+          message: 'Report generator is not available'
+        })
+      }
+
+      const report = await reportGeneratorService.getReportForMap(input.localeIso, input.mode, input.map)
+
+      if (report == undefined) {
+        throw new TRPCError({
+          code: 'NOT_FOUND',
+          message: 'No report available'
+        })
+      }
+
+      const reportHtml = new MarkdownIt().render(report.markdown)
+
+      // cache for 1 hour
+      ctx.res.set('Cache-Control', 'public, max-age=3600')
+      return {
+        html: reportHtml,
+        timestamp: report.timestamp,
+      }
+    }),
+})

web/api/services/FandomService.ts

@@ -0,0 +1,75 @@
+import wtf from 'wtf_wikipedia'
+// @ts-ignore
+import wtfPluginApi from 'wtf-plugin-api'
+import { Cache } from '~/lib/cache'
+
+wtf.extend(wtfPluginApi)
+const wtfOpts = { domain: 'brawlstars.fandom.com', path: 'api.php' }
+
+const FANDOM_CACHE_MINUTES = 60 * 24
+
+export interface FandomModeData {
+  attribution: string
+  shortDescription: string
+  fullDescription: string
+  tips: string[]
+  brawlerTips: Record<string, string>
+}
+
+export default class FandomService {
+  private modeCache = new Cache<string, FandomModeData|undefined>(FANDOM_CACHE_MINUTES)
+
+  async cachedGetModeData(modeName: string) {
+    return await this.modeCache.getOrUpdate(modeName, async () => {
+      console.log('Updating fandom mode cache for', modeName)
+
+      const data = await this.getModeData(modeName)
+      if (data == undefined) {
+        console.log('No fandom mode data for', modeName)
+      }
+
+      return data
+    })
+  }
+
+  private async getModeData(modeName: string): Promise<FandomModeData|undefined> {
+    const modePage: any = await wtf.fetch(modeName, wtfOpts)
+    if (modePage == null) {
+      // does not exist
+      return undefined
+    }
+
+    const description: string = modePage.section('').text()
+    const shortDescription: string = description.split('\n')[0].replaceAll('"', '')
+    const fullDescription: string = description.split('\n')[0]
+
+    const brawlerTips: Record<string, string> = Object.fromEntries(modePage
+      .section('Useful Brawlers')
+      .list(0)
+      .lines()
+      .map((l: any) => l.text())
+      .map((l: string) => {
+        const groups = l.match(/^(\w+): (.*)$/)
+        if (groups == null) {
+          return []
+        }
+        return [groups[1], groups[2]]
+      })
+      .filter((l: string[]) => l.length > 0)
+    )
+
+    const tips = modePage
+      .section('Tips')
+      .list(0)
+      .lines()
+      .map((l: any) => l.text())
+
+    return {
+      attribution: modePage.url(),
+      shortDescription,
+      fullDescription,
+      tips,
+      brawlerTips,
+    }
+  }
+}