Implement DIPPER parrot

GU-CLASP · Apr 23, 2024 · f6da6e0 · f6da6e0
1 parent af383d9
commit f6da6e0
Show file tree

Hide file tree

Showing 11 changed files with 4,592 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/.yarnrc.yml b/.yarnrc.yml
@@ -0,0 +1 @@
+nodeLinker: node-modules
diff --git a/README.org b/README.org
@@ -0,0 +1,22 @@
+* SISU: [[https://github.com/vladmaraev/speechstate][SpeechState]] meets ISU
+
+Implementation based on: Kronlid, F., & Lager, T. (2007, May). [[https://archive.illc.uva.nl/semdial/proceedings/semdial2007_decalog_proceedings.pdf][Implementing the information-state update approach to dialogue management in a slightly extended SCXML]]. In Proceedings of the 11th International Workshop on the Semantics and Pragmatics of Dialogue (DECALOG) (pp. 99-106).
+
+
+How to run:
+1. Install NodeJS (LTS version) and Yarn (v2 or higher)
+2. Install all dependencies:
+   #+begin_src 
+     yarn
+   #+end_src
+3. Create a file =src/azure.ts= with the following contents:
+   #+begin_src 
+     export const KEY = "paste your AZURE KEY here";
+   #+end_src
+4. Run the project:
+   #+begin_src 
+     yarn dev
+   #+end_src
+5. Open the link that was shown in your browser, e.g. http://localhost:5173/
+6. Allow access to you microphone.
+7. When you unblock the pop-up window, and reload the page you will see the state inspector; this can be useful during development.
diff --git a/index.html b/index.html
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + TS</title>
+  </head>
+  <body>
+    <div id="app"></div>
+    <script type="module" src="/src/ui.ts"></script>
+  </body>
+</html>
diff --git a/package.json b/package.json
@@ -0,0 +1,19 @@
+{
+  "name": "sisu",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "preview": "vite preview"
+  },
+  "devDependencies": {
+    "typescript": "^5.2.2",
+    "vite": "^5.2.0"
+  },
+  "dependencies": {
+    "@statelyai/inspect": "^0.2.5",
+    "speechstate": "^2.0.5"
+  }
+}
diff --git a/src/main.ts b/src/main.ts
@@ -0,0 +1,169 @@
+import { assign, createActor, setup, AnyMachineSnapshot } from "xstate";
+import { speechstate } from "speechstate";
+import { createBrowserInspector } from "@statelyai/inspect";
+import { KEY } from "./azure";
+
+const inspector = createBrowserInspector();
+
+const azureCredentials = {
+  endpoint:
+    "https://northeurope.api.cognitive.microsoft.com/sts/v1.0/issuetoken",
+  key: KEY,
+};
+
+const settings = {
+  azureCredentials: azureCredentials,
+  asrDefaultCompleteTimeout: 0,
+  asrDefaultNoInputTimeout: 5000,
+  locale: "en-US",
+  ttsDefaultVoice: "en-US-DavisNeural",
+};
+
+const dmMachine = setup({
+  actions: {
+    /** speak and listen */
+    speak_output_top: ({ context }) =>
+      context.ssRef.send({
+        type: "SPEAK",
+        value: {
+          utterance: context.is.output[0],
+        },
+      }),
+    listen: ({ context }) =>
+      context.ssRef.send({
+        type: "LISTEN",
+      }),
+
+    /** update rules */
+    enqueue_recognition_result: assign(({ context, event }) => {
+      const newIS = {
+        ...context.is,
+        input: [event.value[0].utterance, ...context.is.input],
+      };
+      console.log("[IS enqueue_recognition_result]", newIS);
+      return { is: newIS };
+    }),
+    enqueue_input_timeout: assign(({ context }) => {
+      const newIS = {
+        ...context.is,
+        input: ["timeout", ...context.is.input],
+      };
+      console.log("[IS enqueue_input_timeout]", newIS);
+      return { is: newIS };
+    }),
+    dequeue_input: assign(({ context }) => {
+      const newIS = {
+        ...context.is,
+        input: context.is.input.slice(1),
+      };
+      console.log("[IS dequeue_input]", newIS);
+      return { is: newIS };
+    }),
+    dequeue_output: assign(({ context }) => {
+      const newIS = { ...context.is, output: context.is.output.slice(1) };
+      console.log("[IS dequeue_output]", newIS);
+      return { is: newIS };
+    }),
+    enqueue_output_from_input: assign(({ context }) => {
+      const newIS = {
+        ...context.is,
+        output: [context.is.input[0], ...context.is.output],
+      };
+      console.log("[IS enqueue_output_from_input]", newIS);
+      return { is: newIS };
+    }),
+  },
+  guards: {
+    /** preconditions */
+    lastInputIsTimeout: ({ context }) => context.is.input[0] === "timeout",
+    inputIsNotEmpty: ({ context }) => !!context.is.input[0],
+    outputIsNotEmpty: ({ context }) => !!context.is.output[0],
+  },
+  types: {} as {
+    context: {
+      ssRef?: any;
+      is: { input: string[]; output: string[] };
+    };
+  },
+}).createMachine({
+  context: {
+    is: { input: ["ping"], output: [] },
+  },
+  id: "DM",
+  initial: "Prepare",
+  states: {
+    Prepare: {
+      entry: [
+        assign({
+          ssRef: ({ spawn }) => spawn(speechstate, { input: settings }),
+        }),
+        ({ context }) => context.ssRef.send({ type: "PREPARE" }),
+      ],
+      on: { ASRTTS_READY: "WaitToStart" },
+    },
+    WaitToStart: {
+      on: {
+        CLICK: "Main",
+      },
+    },
+    Main: {
+      initial: "Process",
+      states: {
+        Process: {
+          always: [
+            {
+              guard: "lastInputIsTimeout",
+              actions: "dequeue_input",
+            },
+            {
+              guard: "inputIsNotEmpty",
+              actions: ["enqueue_output_from_input", "dequeue_input"],
+            },
+            {
+              guard: "outputIsNotEmpty",
+              actions: ["speak_output_top", "dequeue_output"],
+            },
+          ],
+          on: { SPEAK_COMPLETE: "Ask" },
+        },
+        Ask: {
+          entry: "listen",
+          on: {
+            RECOGNISED: {
+              target: "Process",
+              actions: "enqueue_recognition_result",
+            },
+            ASR_NOINPUT: {
+              target: "Process",
+              actions: "enqueue_input_timeout",
+            },
+          },
+        },
+      },
+    },
+  },
+});
+
+export const dmActor = createActor(dmMachine, {
+  inspect: inspector.inspect,
+}).start();
+
+let is = dmActor.getSnapshot().context.is;
+console.log("[IS (initial)]", is);
+dmActor.subscribe((snapshot: AnyMachineSnapshot) => {
+  /* if you want to log some parts of the state */
+
+  // is !== snapshot.context.is && console.log("[IS]", snapshot.context.is);
+  is = snapshot.context.is;
+});
+
+export function setupButton(element: HTMLElement) {
+  element.addEventListener("click", () => {
+    dmActor.send({ type: "CLICK" });
+  });
+  dmActor
+    .getSnapshot()
+    .context.ssRef.subscribe((snapshot: AnyMachineSnapshot) => {
+      element.innerHTML = `${Object.values(snapshot.getMeta())[0]["view"]}`;
+    });
+}
diff --git a/src/style.css b/src/style.css
@@ -0,0 +1,96 @@
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+  line-height: 1.5;
+  font-weight: 400;
+
+  color-scheme: light dark;
+  color: rgba(255, 255, 255, 0.87);
+  background-color: #242424;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+a {
+  font-weight: 500;
+  color: #646cff;
+  text-decoration: inherit;
+}
+a:hover {
+  color: #535bf2;
+}
+
+body {
+  margin: 0;
+  display: flex;
+  place-items: center;
+  min-width: 320px;
+  min-height: 100vh;
+}
+
+h1 {
+  font-size: 3.2em;
+  line-height: 1.1;
+}
+
+#app {
+  max-width: 1280px;
+  margin: 0 auto;
+  padding: 2rem;
+  text-align: center;
+}
+
+.logo {
+  height: 6em;
+  padding: 1.5em;
+  will-change: filter;
+  transition: filter 300ms;
+}
+.logo:hover {
+  filter: drop-shadow(0 0 2em #646cffaa);
+}
+.logo.vanilla:hover {
+  filter: drop-shadow(0 0 2em #3178c6aa);
+}
+
+.card {
+  padding: 2em;
+}
+
+.read-the-docs {
+  color: #888;
+}
+
+button {
+  border-radius: 8px;
+  border: 1px solid transparent;
+  padding: 0.6em 1.2em;
+  font-size: 1em;
+  font-weight: 500;
+  font-family: inherit;
+  background-color: #1a1a1a;
+  cursor: pointer;
+  transition: border-color 0.25s;
+}
+button:hover {
+  border-color: #646cff;
+}
+button:focus,
+button:focus-visible {
+  outline: 4px auto -webkit-focus-ring-color;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    color: #213547;
+    background-color: #ffffff;
+  }
+  a:hover {
+    color: #747bff;
+  }
+  button {
+    background-color: #f9f9f9;
+  }
+}
diff --git a/src/ui.ts b/src/ui.ts
@@ -0,0 +1,11 @@
+import { dmActor, setupButton } from "./main";
+
+document.querySelector("#app")!.innerHTML = `
+  <div>
+    <div class="card">
+      <button id="counter" type="button"></button>
+    </div>
+  </div>
+`;
+
+setupButton(document.querySelector("#counter"));
diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts
@@ -0,0 +1 @@
+/// <reference types="vite/client" />
diff --git a/tsconfig.json b/tsconfig.json
@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "module": "ESNext",
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src"]
+}