Pretalx support (#190)

* Update fast-xml dep * Update lock * Start adding pretalx work * Factor out commonly used function * Reimplement pretalx parser for JSON data * Implement minimal pretalx API client * Refactor pretalx backend * Require an access token and URL for pretalx. * Rewrite some imports * Fix yarn.lock * Add tests for pretalx * force ipv4 * Remove stray newline * Use qaEnabled * Licence headers * Pull the list of talks and iterate on those instead. * Update pentabarf to look at online_qa * Add tests for online_qa * Add support for pulling in penta format schemas into the pretalx backend. * Update for code usage * Update short term refresh function to be optional. * Update code with new FOSDEM pretalx extensions * Add extension enum * Add tests * Replace example. * More tidying * Drop unused comment.
matrix-org · Jan 9, 2024 · 864ec25 · 864ec25
1 parent 302d615
commit 864ec25
Show file tree

Hide file tree

Showing 24 changed files with 1,465 additions and 52 deletions.
diff --git a/config/default.yaml b/config/default.yaml
@@ -122,6 +122,11 @@ conference:
   name: "FOSDEM 2021"
 
 
+  # You must specifiy a backend for your schedule.
+  # It may either be "json", "penta", or "pretalx". Remember to only
+  # specify *one*.
+
+  # sample schedule configuration for using the json backend.
   schedule:
     # the backend to pull the schedule from - this can either be a JSON schedule file, or a URL to pull XML from
     # Possible values are "json" or "penta". If JSON is chosen, the path to the file must be provided via the
@@ -131,15 +136,31 @@ conference:
     # the JSON schema of the JSON schedule format.
     scheduleDefinition: "path/to/local/file"
 
-  # sample schedule configuration for using the penta backend. When using this configuration ensure that the json
-  # example above is commented out
+  # sample schedule configuration for using the penta backend.
   # schedule:
     # backend: "penta"
     # The URL to the XML which is updated with conference information.
     # This is read and parsed by the bot during the early stages of
     # setting up the conference.
     # scheduleDefinition: "https://fosdem.org/2021/schedule/xml"
 
+  # sample schedule configuration for using the pretalx backend. 
+  # schedule:
+    # backend: "pretalx"
+    # The format the scheduleDefinition is in. Use "pretalx" if you are loading the
+    # JSON export from pretalx directly, or "fosdem" for the specific pentabarf-xml-format
+    # they use instead.
+    # scheduleFormat: "fosdem"|"pretalx"
+    # The URL to the XML which is updated with conference information.
+    # This is read and parsed by the bot during the early stages of
+    # setting up the conference.
+    # scheduleDefinition: "https://fosdem.org/2021/schedule/xml"
+    # The endpoint to reach the base API path of the conference.
+    # pretalxApiEndpoint: "https://pretalx.example.com/api/events/example-2021/"
+    # Access token for accessing the Pretalx API
+    # pretalxAccessToken: "secret!"
+
+
   # The timezone that the the bot's database is operating off of.
   timezone: "Europe/Brussels"
 

diff --git a/package.json b/package.json
@@ -27,7 +27,7 @@
     "await-lock": "^2.1.0",
     "config": "^3.3.3",
     "express": "^4.17.1",
-    "fast-xml-parser": "^3.17.6",
+    "fast-xml-parser": "^4.3.2",
     "hls.js": "^0.14.17",
     "irc-upd": "^0.11.0",
     "js-yaml": "^3.14.1",

diff --git a/src/Scheduler.ts b/src/Scheduler.ts
@@ -206,7 +206,7 @@ export class Scheduler {
                     // Rationale: Sometimes schedules get changed at short notice, so we try our best to accommodate that.
                     // Rationale for adding 1 minute: so we don't cut it too close to the wire; whilst processing the refresh,
                     //     time may slip forward.
-                    await this.conference.backend.refreshShortTerm((minVar + 1) * 60);
+                    await this.conference.backend.refreshShortTerm?.((minVar + 1) * 60);
                 } catch (e) {
                     LogService.error("Scheduler", `Failed short-term schedule refresh: ${e.message ?? e}\n${e.stack ?? '?'}`);
                 }

diff --git a/src/__tests__/backends/penta/CachingBackend.test.ts b/src/__tests__/backends/penta/CachingBackend.test.ts
@@ -1,7 +1,7 @@
 import { PentaDb } from "../../../backends/penta/db/PentaDb";
 import { test, expect, jest } from "@jest/globals";
 import { PentabarfParser } from "../../../backends/penta/PentabarfParser";
-import { IPentaDbConfig, IPentaScheduleBackendConfig, IPrefixConfig } from "../../../config";
+import { IPrefixConfig } from "../../../config";
 import { PentaBackend } from "../../../backends/penta/PentaBackend";
 import { Role } from "../../../models/schedule";
 import { CachingBackend } from "../../../backends/CachingBackend";

diff --git a/src/__tests__/backends/penta/PentaBackend.test.ts b/src/__tests__/backends/penta/PentaBackend.test.ts
@@ -1,7 +1,7 @@
 import { PentaDb } from "../../../backends/penta/db/PentaDb";
 import { test, expect, jest } from "@jest/globals";
 import { PentabarfParser } from "../../../backends/penta/PentabarfParser";
-import { IPentaDbConfig, IPentaScheduleBackendConfig, IPrefixConfig } from "../../../config";
+import { IPrefixConfig } from "../../../config";
 import { PentaBackend } from "../../../backends/penta/PentaBackend";
 import { Role } from "../../../models/schedule";
 

diff --git a/src/__tests__/backends/penta/PentabarfParser.test.ts b/src/__tests__/backends/penta/PentabarfParser.test.ts
@@ -24,12 +24,16 @@ const prefixConfig: IPrefixConfig = {
     },
 };
 
+function getFixture(fixtureFile: string) {
+    return fs.readFileSync(path.join(__dirname, fixtureFile), 'utf8')
+}
+
 /*
  * A somewhat vague test that just loads in a basic file and checks a few things, comparing against the snapshots in
  * __snapshots__.
  */
 test('parsing pentabarf XML: overview', () => {
-    const xml = fs.readFileSync(path.join(__dirname, "pentabarf01_overview.xml"), 'utf8');
+    const xml = getFixture("pentabarf01_overview.xml");
     const p = new PentabarfParser(xml, prefixConfig);
 
     // TODO the auditorium id and slug look dodgy and not id-like or slug-like...
@@ -49,18 +53,24 @@ test('parsing pentabarf XML: overview', () => {
 
 
 test('duplicate events lead to errors', () => {
-    const xml = fs.readFileSync(path.join(__dirname, "pentabarf03_duplicate_talk.xml"), 'utf8');
+    const xml = getFixture("pentabarf03_duplicate_talk.xml");
 
     expect(() => new PentabarfParser(xml, prefixConfig)).toThrow(
         "Auditorium A.01 (Someroom): Talk E001: this talk already exists and is defined a second time."
     );
 });
 
 test("unrecognised prefixes don't create rooms", () => {
-    const xml = fs.readFileSync(path.join(__dirname, "pentabarf04_unrecognised_prefix.xml"), 'utf8');
+    const xml = getFixture("pentabarf04_unrecognised_prefix.xml");
     const p = new PentabarfParser(xml, prefixConfig);
 
     expect(p.auditoriums.length).toEqual(0);
     expect(p.talks.length).toEqual(0);
     expect(p.interestRooms.length).toEqual(0);
+});
+
+test("tracks that set a online qa value correctly apply to talks", () => {
+    const xml = getFixture("pentabarf05_online_qa.xml");
+    const p = new PentabarfParser(xml, prefixConfig);
+    expect(p.talks).toMatchSnapshot("talks");
 });
diff --git a/src/__tests__/backends/penta/__snapshots__/PentabarfParser.test.ts.snap b/src/__tests__/backends/penta/__snapshots__/PentabarfParser.test.ts.snap
@@ -16,6 +16,7 @@ exports[`parsing pentabarf XML: overview: auditoriums 1`] = `
         "id": "E001",
         "livestream_endTime": 0,
         "prerecorded": true,
+        "pretalxCode": undefined,
         "qa_startTime": null,
         "slug": "testcon_opening_remarks",
         "speakers": [
@@ -61,6 +62,7 @@ exports[`parsing pentabarf XML: overview: conference 1`] = `
           "id": "E001",
           "livestream_endTime": 0,
           "prerecorded": true,
+          "pretalxCode": undefined,
           "qa_startTime": null,
           "slug": "testcon_opening_remarks",
           "speakers": [
@@ -122,6 +124,7 @@ exports[`parsing pentabarf XML: overview: talks 1`] = `
     "id": "E001",
     "livestream_endTime": 0,
     "prerecorded": true,
+    "pretalxCode": undefined,
     "qa_startTime": null,
     "slug": "testcon_opening_remarks",
     "speakers": [
@@ -147,3 +150,132 @@ exports[`parsing pentabarf XML: overview: talks 1`] = `
   },
 ]
 `;
+
+exports[`tracks that set a online qa value correctly apply to talks: talks 1`] = `
+[
+  {
+    "auditoriumId": "A.01 (Someroom)",
+    "dateTs": 1675468800000,
+    "endTime": 1675502100000,
+    "id": "E001",
+    "livestream_endTime": 0,
+    "prerecorded": true,
+    "pretalxCode": undefined,
+    "qa_startTime": 0,
+    "slug": "testcon_opening_remarks",
+    "speakers": [
+      {
+        "email": "",
+        "id": "9012",
+        "matrix_id": "",
+        "name": "John Teststone",
+        "role": "speaker",
+      },
+      {
+        "email": "",
+        "id": "90367",
+        "matrix_id": "",
+        "name": "Evå 完牲 Exampleson",
+        "role": "speaker",
+      },
+    ],
+    "startTime": 1675501200000,
+    "subtitle": "",
+    "title": "Testcon01: Opening Remarks",
+    "track": "TrackWithQA",
+  },
+  {
+    "auditoriumId": "A.01 (Someroom)",
+    "dateTs": 1675468800000,
+    "endTime": 1675503900000,
+    "id": "E002",
+    "livestream_endTime": 0,
+    "prerecorded": true,
+    "pretalxCode": undefined,
+    "qa_startTime": 0,
+    "slug": "testcon_opening_remarks",
+    "speakers": [
+      {
+        "email": "",
+        "id": "9012",
+        "matrix_id": "",
+        "name": "John Teststone",
+        "role": "speaker",
+      },
+      {
+        "email": "",
+        "id": "90367",
+        "matrix_id": "",
+        "name": "Evå 完牲 Exampleson",
+        "role": "speaker",
+      },
+    ],
+    "startTime": 1675502100000,
+    "subtitle": "",
+    "title": "Testcon01: Opening Remarks",
+    "track": "TrackWithoutQA",
+  },
+  {
+    "auditoriumId": "A.01 (Someroom)",
+    "dateTs": 1675468800000,
+    "endTime": 1675504800000,
+    "id": "E003",
+    "livestream_endTime": 0,
+    "prerecorded": true,
+    "pretalxCode": undefined,
+    "qa_startTime": null,
+    "slug": "testcon_opening_remarks",
+    "speakers": [
+      {
+        "email": "",
+        "id": "9012",
+        "matrix_id": "",
+        "name": "John Teststone",
+        "role": "speaker",
+      },
+      {
+        "email": "",
+        "id": "90367",
+        "matrix_id": "",
+        "name": "Evå 完牲 Exampleson",
+        "role": "speaker",
+      },
+    ],
+    "startTime": 1675503000000,
+    "subtitle": "",
+    "title": "Testcon01: Opening Remarks",
+    "track": "UnspecifiedTrack",
+  },
+  {
+    "auditoriumId": "AQ.5 (QA Room)",
+    "dateTs": 1675468800000,
+    "endTime": 1675505700000,
+    "id": "E004",
+    "livestream_endTime": 0,
+    "prerecorded": true,
+    "pretalxCode": undefined,
+    "qa_startTime": 0,
+    "slug": "testcon_opening_remarks",
+    "speakers": [
+      {
+        "email": "",
+        "id": "9012",
+        "matrix_id": "",
+        "name": "John Teststone",
+        "role": "speaker",
+      },
+      {
+        "email": "",
+        "id": "90367",
+        "matrix_id": "",
+        "name": "Evå 完牲 Exampleson",
+        "role": "speaker",
+      },
+    ],
+    "startTime": 1675503900000,
+    "subtitle": "",
+    "title": "Testcon01: Opening Remarks",
+    "track": "UnspecifiedTrack",
+  },
+]
+`;
diff --git a/src/__tests__/backends/penta/pentabarf05_online_qa.xml b/src/__tests__/backends/penta/pentabarf05_online_qa.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schedule>
+  <conference>
+    <title>Testcon01</title>
+    <subtitle/>
+  </conference>
+  <tracks>
+    <track online_qa="true">TrackWithQA</track>
+    <track online_qa="false">TrackWithoutQA</track>
+  </tracks>
+  <day index="1" date="2023-02-04">
+    <room name="A.01 (Someroom)">
+      <event id="E001">
+        <start>09:00</start>
+        <duration>00:15</duration>
+        <room>A.01 (Someroom)</room>
+        <slug>testcon_opening_remarks</slug>
+        <title>Testcon01: Opening Remarks</title>
+        <subtitle/>
+        <track>TrackWithQA</track>
+        <type>devroom</type>
+        <language/>
+        <abstract>
+          &lt;p&gt;Opening remarks&lt;/p&gt;
+        </abstract>
+        <description></description>
+        <persons>
+          <person id="9012">John Teststone</person>
+          <person id="90367">Evå 完牲 Exampleson</person> <!-- seemed wise to test some Unicode -->
+        </persons>
+        <attachments>
+        </attachments>
+        <links>
+          <link href="https://example.org">Submit feedback</link>
+        </links>
+      </event>
+      <event id="E002">
+        <start>09:15</start>
+        <duration>00:30</duration>
+        <room>A.01 (Someroom)</room>
+        <slug>testcon_opening_remarks</slug>
+        <title>Testcon01: Opening Remarks</title>
+        <subtitle/>
+        <track>TrackWithoutQA</track>
+        <type>devroom</type>
+        <language/>
+        <abstract>
+          &lt;p&gt;Opening remarks&lt;/p&gt;
+        </abstract>
+        <description></description>
+        <persons>
+          <person id="9012">John Teststone</person>
+          <person id="90367">Evå 完牲 Exampleson</person> <!-- seemed wise to test some Unicode -->
+        </persons>
+        <attachments>
+        </attachments>
+        <links>
+          <link href="https://example.org">Submit feedback</link>
+        </links>
+      </event>
+      <event id="E003">
+        <start>09:30</start>
+        <duration>00:30</duration>
+        <room>A.01 (Someroom)</room>
+        <slug>testcon_opening_remarks</slug>
+        <title>Testcon01: Opening Remarks</title>
+        <subtitle/>
+        <track>UnspecifiedTrack</track>
+        <type>devroom</type>
+        <language/>
+        <abstract>
+          &lt;p&gt;Opening remarks&lt;/p&gt;
+        </abstract>
+        <description></description>
+        <persons>
+          <person id="9012">John Teststone</person>
+          <person id="90367">Evå 完牲 Exampleson</person> <!-- seemed wise to test some Unicode -->
+        </persons>
+        <attachments>
+        </attachments>
+        <links>
+          <link href="https://example.org">Submit feedback</link>
+        </links>
+      </event>
+    </room>
+    <room name="AQ.5 (QA Room)">
+      <event id="E004" >
+        <start>09:45</start>
+        <duration>00:30</duration>
+        <room>AQ.5 (QA Room)</room>
+        <slug>testcon_opening_remarks</slug>
+        <title>Testcon01: Opening Remarks</title>
+        <subtitle/>
+        <track>UnspecifiedTrack</track>
+        <type>devroom</type>
+        <language/>
+        <abstract>
+          &lt;p&gt;Opening remarks&lt;/p&gt;
+        </abstract>
+        <description></description>
+        <persons>
+          <person id="9012">John Teststone</person>
+          <person id="90367">Evå 完牲 Exampleson</person> <!-- seemed wise to test some Unicode -->
+        </persons>
+        <attachments>
+        </attachments>
+        <links>
+          <link href="https://example.org">Submit feedback</link>
+        </links>
+      </event>
+    </room>
+  </day>
+</schedule>