diff --git a/Code/appointment.json b/Code/appointment.json new file mode 100644 index 0000000..211cbc6 --- /dev/null +++ b/Code/appointment.json @@ -0,0 +1,467 @@ +{ + "projectFileVersion": "2022-10-01-preview", + "stringIndexType": "Utf16CodeUnit", + "metadata": { + "projectKind": "Conversation", + "settings": { + "confidenceThreshold": 0, + "normalizeCasing": true + }, + "projectName": "appointment", + "multilingual": false, + "description": "", + "language": "en-us" + }, + "assets": { + "projectKind": "Conversation", + "intents": [ + { + "category": "None" + }, + { + "category": "create a meeting" + }, + { + "category": "who is Adam Lambert " + }, + { + "category": "who is adam lambert" + } + ], + "entities": [ + { + "category": "meeting time", + "compositionSetting": "combineComponents", + "list": { + "sublists": [ + { + "listKey": "Week", + "synonyms": [ + { + "language": "en-us", + "values": [ + "Week", + "Friday", + "Monday", + "Tuesday", + "Thursday", + "Wednesday" + ] + } + ] + } + ] + } + }, + { + "category": "meeting title", + "compositionSetting": "combineComponents", + "list": { + "sublists": [ + { + "listKey": "Match", + "synonyms": [ + { + "language": "en-us", + "values": [ + "Match", + "Football", + "Volleyball", + "Computer Game" + ] + } + ] + }, + { + "listKey": "Dinner", + "synonyms": [ + { + "language": "en-us", + "values": [ + "Dinner", + "hot pot", + "soup", + "Chinese Food", + "Indian Food", + "Thailand Food", + "Swedish Food" + ] + } + ] + } + ] + }, + "prebuilts": [ + { + "category": "General.Event" + } + ] + }, + { + "category": "yesorno", + "compositionSetting": "combineComponents", + "list": { + "sublists": [ + { + "listKey": "no", + "synonyms": [ + { + "language": "en-us", + "values": [ + "no", + "no way" + ] + } + ] + }, + { + "listKey": "yes", + "synonyms": [ + { + "language": "en-us", + "values": [ + "yes", + "of course", + "yeah" + ] + } + ] + } + ] + }, + "prebuilts": [ + { + "category": "Choice.Boolean" + } + ] + }, + { + "category": "Age", + "compositionSetting": "combineComponents", + "prebuilts": [ + { + "category": "Quantity.Age" + } + ] + }, + { + "category": "BirthDay", + "compositionSetting": "combineComponents", + "prebuilts": [ + { + "category": "DateTime" + } + ] + }, + { + "category": "Info", + "compositionSetting": "combineComponents", + "list": { + "sublists": [ + { + "listKey": "Career", + "synonyms": [ + { + "language": "en-us", + "values": [ + "Career", + "singer", + "artist" + ] + } + ] + } + ] + } + } + ], + "utterances": [ + { + "text": "Adam Lambert is born in Jan 29th.", + "language": "en-us", + "intent": "who is adam lambert", + "entities": [ + { + "category": "BirthDay", + "offset": 24, + "length": 8 + } + ], + "dataset": "Train" + }, + { + "text": "Adam Lambert is a Singer and Artist", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [ + { + "category": "Info", + "offset": 18, + "length": 6 + }, + { + "category": "Info", + "offset": 29, + "length": 6 + } + ], + "dataset": "Train" + }, + { + "text": "Adam Lambert is born on Jan 29th.", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [ + { + "category": "BirthDay", + "offset": 24, + "length": 8 + } + ], + "dataset": "Train" + }, + { + "text": "Adam Lambert is 42 years old", + "language": "en-us", + "intent": "who is adam lambert", + "entities": [ + { + "category": "Age", + "offset": 16, + "length": 11 + } + ], + "dataset": "Train" + }, + { + "text": "Who is the individual named Adam Lambert?", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [], + "dataset": "Train" + }, + { + "text": "What can you tell me about Adam Lambert?", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [], + "dataset": "Train" + }, + { + "text": "Could you give me some background on Adam Lambert?", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [], + "dataset": "Train" + }, + { + "text": "Who exactly is Adam Lambert?", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [], + "dataset": "Train" + }, + { + "text": "Tell me about the person known as Adam Lambert.", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [], + "dataset": "Train" + }, + { + "text": "What details do you have about Adam Lambert?", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [], + "dataset": "Train" + }, + { + "text": "Can you provide information on Adam Lambert?", + "language": "en-us", + "intent": "who is Adam Lambert ", + "entities": [], + "dataset": "Train" + }, + { + "text": "It would be beneficial to schedule a meeting next Friday.", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 37, + "length": 7 + }, + { + "category": "meeting time", + "offset": 50, + "length": 6 + } + ], + "dataset": "Train" + }, + { + "text": "I'd like to propose a weekly check-in meeting on Monday.", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 29, + "length": 16 + }, + { + "category": "meeting time", + "offset": 49, + "length": 6 + } + ], + "dataset": "Train" + }, + { + "text": "Let's organize a special meeting next Wednesday.", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 17, + "length": 15 + }, + { + "category": "meeting time", + "offset": 38, + "length": 9 + } + ], + "dataset": "Train" + }, + { + "text": "I recommend we convene a cross-functional meeting next Tuesday.", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 25, + "length": 24 + }, + { + "category": "meeting time", + "offset": 55, + "length": 7 + } + ], + "dataset": "Train" + }, + { + "text": "Let's plan a departmental meeting next Friday.", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 13, + "length": 20 + }, + { + "category": "meeting time", + "offset": 39, + "length": 6 + } + ], + "dataset": "Train" + }, + { + "text": "How about setting up a conference call for Thursday?", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 23, + "length": 15 + }, + { + "category": "meeting time", + "offset": 43, + "length": 9 + } + ], + "dataset": "Train" + }, + { + "text": "I suggest creating a meeting on Wednesday.", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 21, + "length": 7 + }, + { + "category": "meeting time", + "offset": 32, + "length": 9 + } + ], + "dataset": "Train" + }, + { + "text": "Let's schedule a virtual meeting for next Thursday.", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 17, + "length": 15 + }, + { + "category": "meeting time", + "offset": 42, + "length": 8 + } + ], + "dataset": "Train" + }, + { + "text": "Can we organize a brief team meeting this Friday?", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 24, + "length": 12 + }, + { + "category": "meeting time", + "offset": 42, + "length": 6 + } + ], + "dataset": "Train" + }, + { + "text": "I propose we set up a meeting next Monday.", + "language": "en-us", + "intent": "create a meeting", + "entities": [ + { + "category": "meeting title", + "offset": 22, + "length": 7 + }, + { + "category": "meeting time", + "offset": 35, + "length": 6 + } + ], + "dataset": "Train" + } + ] + } +} \ No newline at end of file diff --git a/Code/azure.js b/Code/azure.js new file mode 100644 index 0000000..976e8e8 --- /dev/null +++ b/Code/azure.js @@ -0,0 +1,2 @@ +export const KEY = "4e763d7012be49e5a48c5db78d8c33b5"; +export const NLU_KEY = "953bed351641435f9ad2e8198192f38c"; \ No newline at end of file diff --git a/Code/dm.js b/Code/dm.js index 9fb7198..1b1987f 100644 --- a/Code/dm.js +++ b/Code/dm.js @@ -20,32 +20,99 @@ const settings = { }; /* Grammar definition */ -const grammar = { +const grammar_person = { vlad: { person: "Vladislav Maraev" }, aya: { person: "Nayat Astaiza Soriano" }, rasmus: { person: "Rasmus Blanck" }, + marry: { person: "Marry" }, + tom: { person: "Tom" }, + black: { person: "Black" }, + jerry: { person: "Jerry" }, +}; + +const grammar_day = { monday: { day: "Monday" }, tuesday: { day: "Tuesday" }, + wednesday: { day: "Wednesday" }, + thursday: { day: "Thursday" }, + friday: { day: "Friday" }, +}; + +const grammar_time = { + "8": { time: "8:00" }, + "9": { time: "9:00" }, "10": { time: "10:00" }, "11": { time: "11:00" }, + "12": { time: "12:00" }, + "13": { time: "13:00" }, + "14": { time: "14:00" }, + "15": { time: "15:00" }, + "16": { time: "16:00" }, +}; + +const grammar_positive_answer = { + "yes": { answer: "Yes" }, + "of course": { answer: "Of course" }, + "yeah": { answer: "Yeah" }, +}; + +const grammar_negative_answer = { + "no": { answer: "No" }, + "no way": { answer: "No way"}, }; /* Helper functions */ -function isInGrammar(utterance) { - return utterance.toLowerCase() in grammar; +function isInPersonGrammar(utterance) { + return utterance.toLowerCase() in grammar_person; +} + +function isInDayGrammar(utterance) { + return utterance.toLowerCase() in grammar_day; +} + +function isInTimeGrammar(utterance) { + return utterance.toLowerCase() in grammar_time; } function getPerson(utterance) { - return (grammar[utterance.toLowerCase()] || {}).person; + return (grammar_person[utterance.toLowerCase()] || {}).person; +} + +function getDate(utterance) { + return (grammar_day[utterance.toLowerCase()] || {}).day; } +function getTime(utterance) { + return (grammar_time[utterance.toLowerCase()] || {}).time; +} + +function isYes(utterance) { + return utterance.toLowerCase() in grammar_positive_answer; +} + +function isNo(utterance) { + return utterance.toLowerCase() in grammar_negative_answer; +} + + const dmMachine = setup({ actions: { /* define your actions here */ + say: ({ context }, params) => + context.ssRef.send({ + type: "SPEAK", + value: { + utterance: params, + }, + }), }, }).createMachine({ context: { count: 0, + person: undefined, + date: undefined, + time: undefined, + }, id: "DM", initial: "Prepare", @@ -68,11 +135,203 @@ const dmMachine = setup({ initial: "Prompt", states: { Prompt: { + entry:[{type: "say", params:`Let's create an appointment`}], + on: { SPEAK_COMPLETE: "#DM.WithWhom" }, + }, + }, + }, + WithWhom: { + initial: "Who", + id: "who", + states: { + Who: { + entry: [{type: "say", params:`Who are you meeting with?`}], + on: { SPEAK_COMPLETE: "Ask" }, + }, + Ask: { + entry: ({ context }) => + context.ssRef.send({ + type: "LISTEN", + value: { + noInputTimeOut: 1000 + } + }), + on: { + RECOGNISED: + [{ + guard: ({ event }) => isInPersonGrammar(event.value[0].utterance) === true, + actions: [ + ({ context, event }) => { context.person = getPerson(event.value[0].utterance) }, + ], + target: "#DM.WhichDate", + }, + { + guard: ({ event }) => isInPersonGrammar(event.value[0].utterance) === false, + target: "#DM.NotPersonGram", + }], + ASR_NOINPUT:"#DM.NoPersonVoice", + }, + }, + }, + }, + /* Not received voice */ + NoPersonVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WithWhom" + }, + + }, + + }, + }, + /* Not in grammar */ + NotPersonGram: { + initial: "Notgram", + states: { + Notgram: { + entry: [{type: "say", params:`The word is not in Gramma. Please answer the question again`}], + on: { + SPEAK_COMPLETE: "#DM.WithWhom" + }, + }, + + }, + }, + WhichDate: { + initial: "Whichdate", + states: { + Whichdate: { + entry: [{type: "say", params:`On which day is your meeting?`}], + on: { SPEAK_COMPLETE: "Ask" }, + }, + Ask: { + entry: ({ context }) => + context.ssRef.send({ + type: "LISTEN", + value: { + noInputTimeOut: 1000 + } + }), + on: { + RECOGNISED: + [ + { + guard: ({ event }) => isInDayGrammar(event.value[0].utterance) === true, + actions: [ + ({ context, event }) => { context.date = getDate(event.value[0].utterance) }, + ], + target: "#DM.WholeDay", + }, + { + guard: ({ event }) => isInDayGrammar(event.value[0].utterance) === false, + target: "#DM.NotDateGram", + }, + ], + ASR_NOINPUT:"#DM.NoDateVoice", + }, + }, + }, + }, + /* Not received voice */ + NoDateVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WhichDate" + }, + }, + }, + }, + /* Not in grammar */ + NotDateGram: { + initial: "Notgram", + states: { + Notgram: { + entry: [{type: "say", params:`The word is not in Gramma. Please answer the question again`}], + on: { + SPEAK_COMPLETE: "#DM.WhichDate" + }, + }, + }, + }, + WholeDay: { + initial: "Whole", + states: { + Whole: { + + entry: ({ context }) => + context.ssRef.send({ + type: "SPEAK", + value: { + utterance: `Will it take the whole day?`, + }, + }), + on: { SPEAK_COMPLETE: "Ask" }, + }, + Ask: { + entry: ({ context }) => + context.ssRef.send({ + type: "LISTEN", + value: { + noInputTimeOut: 1000 + } + }), + on: { + RECOGNISED: + [{ + guard: ({ event }) => isYes(event.value[0].utterance) === true, + target: "#DM.CreateWholeDay", + }, + { + guard: ({ event }) => isNo(event.value[0].utterance) === true, + target: "#DM.WhatTime", + }, + "#DM.NotWholeDayGram" + ], + ASR_NOINPUT:"#DM.NoWholeDayVoice", + }, + }, + }, + }, + /* Not received voice */ + NoWholeDayVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WholeDay" + }, + }, + }, + }, + /* Not in grammar */ + NotWholeDayGram: { + initial: "Notgram", + states: { + Notgram: { + entry: [{type: "say", params:`The word is not in Gramma. Please answer the question again`}], + on: { + SPEAK_COMPLETE: "#DM.WholeDay" + }, + }, + }, + }, + CreateWholeDay: { + initial: "Createwhole", + states: { + Createwhole: { entry: ({ context }) => context.ssRef.send({ type: "SPEAK", value: { - utterance: `Hello world!`, + utterance: `Do you want me to create an appointment with ${context.person} on ${context.date} for the whole day?`, }, }), on: { SPEAK_COMPLETE: "Ask" }, @@ -81,26 +340,181 @@ const dmMachine = setup({ entry: ({ context }) => context.ssRef.send({ type: "LISTEN", + value: { + noInputTimeOut: 1000 + }}), + on: { + RECOGNISED: + [{ + guard: ({ event }) => isYes(event.value[0].utterance) === true, + target: "#DM.FinishAppointment", + }, + { + guard: ({ event }) => isNo(event.value[0].utterance) === true, + target: "#DM.WithWhom", + }, + "#DM.NotCreateWholeDayGram" + ], + ASR_NOINPUT: "#DM.NoCreateWholeDayVoice", + }, + }, + }, + }, + /* Not received voice */ + NoCreateWholeDayVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.CreateWholeDay" + } + }, + }, + }, + /* Not in grammar */ + NotCreateWholeDayGram: { + initial: "Notgram", + states: { + Notgram: { + entry: [{type: "say", params:`The word is not in Gramma. Please answer the question again`}], + on: { + SPEAK_COMPLETE: "#DM.CreateWholeDay" + }, + }, + }, + }, + WhatTime: { + initial: "Whattime", + states: { + Whattime: { + entry: [{type: "say", params:`What time is your meeting?`}], + on: { SPEAK_COMPLETE: "Ask" }, + }, + Ask: { + entry: ({ context }) => + context.ssRef.send({ + type: "LISTEN", + value: { + noInputTimeOut: 1000 + } }), - on: { - RECOGNISED: { - actions: ({ context, event }) => - context.ssRef.send({ - type: "SPEAK", - value: { - utterance: `You just said: ${ - event.value[0].utterance - }. And it ${ - isInGrammar(event.value[0].utterance) ? "is" : "is not" - } in the grammar.`, - }, - }), - }, - SPEAK_COMPLETE: "#DM.Done", + on: { + RECOGNISED: + [ + { + guard: ({ event }) => isInTimeGrammar(event.value[0].utterance) === true, + actions: [ + ({ context, event }) => { context.time = getTime(event.value[0].utterance) }, + ], + target: "#DM.CreateTime", + }, + { + guard: ({ event }) => isInTimeGrammar(event.value[0].utterance) === false, + target: "#DM.NotTimeGram", + }, + "#DM.NotTimeGram"], + ASR_NOINPUT: "#DM.NoTimeVoice", + }, + }, + }, + }, + /* Not received voice */ + NoTimeVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WhatTime" + }, + }, + }, + }, + /* Not in grammar */ + NotTimeGram: { + initial: "Notgram", + states: { + Notgram: { + entry: [{type: "say", params:`The word is not in Gramma. Please answer the question again`}], + on: { + SPEAK_COMPLETE: "#DM.WhatTime" + }, + }, + }, + }, + CreateTime: { + initial: "Createtime", + states: { + Createtime: { + entry: ({ context }) => + context.ssRef.send({ + type: "SPEAK", + value: { + utterance: `Do you want me to create an appointment with ${context.person} on ${context.date} at ${context.time}?`, + }, + }), + on: { SPEAK_COMPLETE: "Ask" }, + }, + Ask: { + entry: ({ context }) => + context.ssRef.send({ + type: "LISTEN", + value: { + noInputTimeOut: 1000 + + } + }), + on: { + RECOGNISED: + [{ + guard: ({ event }) => isYes(event.value[0].utterance) === true, + target: "#DM.FinishAppointment", + }, + { + guard: ({ event }) => isNo(event.value[0].utterance) === true, + target: "#DM.WithWhom", + }, + "#DM.NotCreateTimeGram" + ], + ASR_NOINPUT: "#DM.NoCreateTimeVoice", }, }, }, }, + /* Not received voice */ + NoCreateTimeVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.CreateTime" + }, + }, + }, + }, + /* Not in grammar */ + NotCreateTimeGram: { + initial: "Notgram", + states: { + Notgram: { + entry: [{type: "say", params:`The word is not in Gramma. Please answer the question again`}], + on: { + SPEAK_COMPLETE: "#DM.CreateTime" + }, + }, + }, + }, + FinishAppointment: { + initial: "Finishappointment", + states: { + Finishappointment: { + entry: [{type: "say", params:`Your appointment has been created!`}], + on: { SPEAK_COMPLETE: "#DM.Done" }, + }, + }, + }, Done: { on: { CLICK: "PromptAndAsk", @@ -117,11 +531,22 @@ dmActor.subscribe((state) => { /* if you want to log some parts of the state */ }); +/*SetTimeout for start automatically*/ export function setupButton(element) { - element.addEventListener("click", () => { - dmActor.send({ type: "CLICK" }); - }); - dmActor.getSnapshot().context.ssRef.subscribe((snapshot) => { - element.innerHTML = `${snapshot.value.AsrTtsManager.Ready}`; + element.addEventListener("click", + () => { + setTimeout(function(){ + dmActor.send({ type: "CLICK" });}, 100) }); + setTimeout(function(){ + dmActor.send({ type: "CLICK" });}, 1000); + setTimeout(function(){ + + dmActor.getSnapshot().context.ssRef.subscribe((snapshot) => { + element.innerHTML = `${snapshot.value.AsrTtsManager.Ready}`; + }); + + }, 10) + } + diff --git a/Code/dm3.js b/Code/dm3.js new file mode 100644 index 0000000..9eb7a5f --- /dev/null +++ b/Code/dm3.js @@ -0,0 +1,128 @@ +import { assign, createActor, setup } from "xstate"; +import { speechstate } from "speechstate"; +import { createBrowserInspector } from "@statelyai/inspect"; +import { KEY } from "./azure.js"; + +const inspector = createBrowserInspector(); + +const azureCredentials = { + endpoint: + "https://northeurope.api.cognitive.microsoft.com/sts/v1.0/issuetoken", + key: KEY, +}; + +const settings = { + azureCredentials: azureCredentials, + speechRecognitionEndpointId: "758aa8a0-93d1-4aa7-a377-28179ce8881b", + asrDefaultCompleteTimeout: 0, + asrDefaultNoInputTimeout: 5000, + locale: "en-US", + ttsDefaultVoice: "en-US-DavisNeural", +}; + +/* Grammar definition */ +const grammar = { + vlad: { person: "Vladislav Maraev" }, + aya: { person: "Nayat Astaiza Soriano" }, + rasmus: { person: "Rasmus Blanck" }, + monday: { day: "Monday" }, + tuesday: { day: "Tuesday" }, + "10": { time: "10:00" }, + "11": { time: "11:00" }, +}; + +/* Helper functions */ +function isInGrammar(utterance) { + return utterance.toLowerCase() in grammar; +} + +function getPerson(utterance) { + return (grammar[utterance.toLowerCase()] || {}).person; +} + +const dmMachine = setup({ + actions: { + /* define your actions here */ + }, +}).createMachine({ + context: { + count: 0, + }, + id: "DM", + initial: "Prepare", + states: { + Prepare: { + entry: [ + assign({ + ssRef: ({ spawn }) => spawn(speechstate, { input: settings }), + }), + ({ context }) => context.ssRef.send({ type: "PREPARE" }), + ], + on: { ASRTTS_READY: "WaitToStart" }, + }, + WaitToStart: { + on: { + CLICK: "PromptAndAsk", + }, + }, + PromptAndAsk: { + initial: "Prompt", + states: { + Prompt: { + entry: ({ context }) => + context.ssRef.send({ + type: "SPEAK", + value: { + utterance: `Hello world!`, + }, + }), + on: { SPEAK_COMPLETE: "Ask" }, + }, + Ask: { + entry: ({ context }) => + context.ssRef.send({ + type: "LISTEN", + }), + on: { + RECOGNISED: { + actions: ({ context, event }) => + context.ssRef.send({ + type: "SPEAK", + value: { + utterance: `You just said: ${ + event.value[0].utterance + }. And it ${ + isInGrammar(event.value[0].utterance) ? "is" : "is not" + } in the grammar.`, + }, + }), + }, + SPEAK_COMPLETE: "#DM.Done", + }, + }, + }, + }, + Done: { + on: { + CLICK: "PromptAndAsk", + }, + }, + }, +}); + +const dmActor = createActor(dmMachine, { + inspect: inspector.inspect, +}).start(); + +dmActor.subscribe((state) => { + /* if you want to log some parts of the state */ +}); + +export function setupButton(element) { + element.addEventListener("click", () => { + dmActor.send({ type: "CLICK" }); + }); + dmActor.getSnapshot().context.ssRef.subscribe((snapshot) => { + element.innerHTML = `${snapshot.value.AsrTtsManager.Ready}`; + }); +} \ No newline at end of file diff --git a/Code/dm4.js b/Code/dm4.js new file mode 100644 index 0000000..2f00acc --- /dev/null +++ b/Code/dm4.js @@ -0,0 +1,347 @@ +import { assign, createActor, setup } from "xstate"; +import { speechstate } from "speechstate"; +import { createBrowserInspector } from "@statelyai/inspect"; +import { KEY } from "./azure.js"; +import { NLU_KEY } from "./azure.js"; +/* +Regarding improveingNLU coverage, I think I need to add more time datas and person datas to the model. +*/ + const inspector = createBrowserInspector(); + const azureLanguageCredentials = { + endpoint: "https://languageresource58888.cognitiveservices.azure.com/language/:analyze-conversations?api-version=2022-10-01-preview" /** your Azure CLU prediction URL */, + key: NLU_KEY /** reference to your Azure CLU key */, + deploymentName: "appointment" /** your Azure CLU deployment */, + projectName: "appointment" /** your Azure CLU project name */, + }; + const azureCredentials = { + endpoint: + "https://northeurope.api.cognitive.microsoft.com/sts/v1.0/issuetoken", + key: KEY, + }; + const settings = { + azureLanguageCredentials: azureLanguageCredentials /** global activation of NLU */, + azureCredentials: azureCredentials, + asrDefaultCompleteTimeout: 0, + asrDefaultNoInputTimeout: 5000, + locale: "en-US", + ttsDefaultVoice: "en-US-DavisNeural", + }; + + const dmMachine = setup({ + actions: { + /* define your actions here */ + say: ({ context }, params) => + context.ssRef.send({ + type: "SPEAK", + value: { + utterance: params, + }, + }), + listen:({ context }) => + context.ssRef.send({ + type: "LISTEN", + value: { nlu: true } /** Local activation of NLU */, + }), + }, + }).createMachine({ + context: { + count: 0, + // for Who is X part + age: undefined, + birthday: undefined, + + // for creating meeting part + weekdate: undefined, + meeting_name: undefined, + + }, + id: "DM", + initial: "Prepare", + states: { + Prepare: { + entry: [ + assign({ + ssRef: ({ spawn }) => spawn(speechstate, { input: settings }), + }), + ({ context }) => context.ssRef.send({ type: "PREPARE" }), + ], + on: { ASRTTS_READY: "WaitToStart" }, + }, + WaitToStart: { + on: { + CLICK: "PromptAndAsk", + }, + }, + PromptAndAsk: { + initial: "Prompt", + states: { + Prompt: { + entry:[{type: "say", params:`Let's start conversation`}], + on: { SPEAK_COMPLETE: "#DM.ProposeQ" }, + }, + }, + }, + /* jump to one of two questions */ + ProposeQ: { + initial: "proposeQ", + states: { + proposeQ: { + entry: [{type: "say", params:`What's your question?`}], + on: { SPEAK_COMPLETE: "Ask" }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [{ + guard: ({ event }) => event.nluValue.topIntent === "create a meeting", + target: "#DM.WhichDate", + }, + { + guard: ({ event }) => event.nluValue.topIntent === "who is adam lambert" , + target: "#DM.WhoisAdam", + }, + "#DM.NoQVoice" + ], + ASR_NOINPUT:"#DM.NoQVoice", + }, + }, + }, + }, + /* Not received voice */ + NoQVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.ProposeQ" + }, + }, + }, + }, + + /* Meeting Time */ + //Answer e.g. Monday to Friday + WhichDate: { + initial: "whichDate", + states: { + whichDate: { + entry: [{type: "say", params:`On which date is the meeting?`}], + on: { + SPEAK_COMPLETE: "Ask" + }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + //e.g. Adam Lambert is 42 years old + guard: ({ event }) => event.nluValue.entities[0].text === "Monday" || "Tuesday" || "Wednesday" || "Thursday" || "Friday", + actions: [ + ({ context, event }) => { context.weekdate = event.nluValue.entities[0].text}, + ], + target: "#DM.MeetingTitle", + }, + "#DM.NoWhichDateVoice" + ], + ASR_NOINPUT:"#DM.NoWhichDateVoice", + }, + }, + }, + }, + /* Not received voice */ + NoWhichDateVoice: { + initial: "noWhichDateVoice", + states: { + noWhichDateVoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WhichDate" + }, + }, + }, + }, + /* Meeting Title */ + //Answer e.g. "departmental meeting" || "meeting" || "virtual meeting" + MeetingTitle: { + initial: "meetingTitle", + states: { + meetingTitle: { + entry: [{type: "say", params:`What's the meeting title?`}], + on: { + SPEAK_COMPLETE: "Ask" + }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + //e.g. Adam Lambert is 42 years old + guard: ({ event }) => event.nluValue.entities[0].text === "departmental meeting" || "meeting" || "virtual meeting", + actions: [ + ({ context, event }) => { context.meeting_name = event.nluValue.entities[0].text}, + ], + target: "#DM.MeetingInfo", + }, + "#DM.NoMeetingTitleVoice" + ], + ASR_NOINPUT:"#DM.NoMeetingTitleVoice", + }, + }, + }, + }, + /* Not received voice */ + NoMeetingTitleVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.MeetingTitle" + }, + }, + }, + }, + /* Repeat the Info of the Meeting */ + MeetingInfo: { + initial: "adamInfo", + states: { + adamInfo: { + entry: [{type: "say", params: ({context}) => `The meeting is on ${context.weekdate} and the meeting title is ${context.meeting_name} `}], + on: { + SPEAK_COMPLETE: "#DM.Done" + }, + }, + }, + }, + + + /* Who is Adam Lambert */ + //Answer e.g. Adam Lambert is 42 years old + WhoisAdam: { + initial: "whoisAdam", + states: { + whoisAdam: { + entry: [{type: "say", params:`How old is Adam Lambert?`}], + on: { + SPEAK_COMPLETE: "Ask" + }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + //e.g. Adam Lambert is 42 years old + guard: ({ event }) => event.nluValue.entities[0].text === "42 years old", + actions: [ + ({ context, event }) => { context.age = event.nluValue.entities[0].text}, + ], + target: "#DM.WhatAdamBirth", + }, + "#DM.NoAdamVoice" + ], + ASR_NOINPUT:"#DM.NoAdamVoice", + }, + }, + }, + }, + /* Not received voice */ + NoAdamVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WhoisAdam" + }, + }, + }, + }, + /* When is Adam Lambert's birthday */ + //Answer e.g. Adam Lambert is born in Jan 29th + WhatAdamBirth: { + initial: "whatAdamBirth", + states: { + whatAdamBirth: { + entry: [{type: "say", params:`When is Adam Lambert's birthday?`}], + on: { + SPEAK_COMPLETE: "Ask" + }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + //e.g. Adam Lambert is born in Jan 29th + guard: ({ event }) => event.nluValue.entities[0].text === "Jan 29th", + actions: [ + ({ context, event }) => { context.birthday = event.nluValue.entities[0].text}, + ], + target: "#DM.AdamInfo", + }, + "#DM.NoAdamBirthDayVoice" + ], + ASR_NOINPUT:"#DM.NoAdamBirthDayVoice", + }, + }, + }, + }, + /* Not received voice */ + NoAdamBirthDayVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WhatAdamBirth" + }, + }, + }, + }, + /* Repeat the Info of the Person */ + AdamInfo: { + initial: "adamInfo", + states: { + adamInfo: { + entry: [{type: "say", params: ({context}) => `Adam Lambert is ${context.age} and his birthday is ${context.birthday} `}], + on: { + SPEAK_COMPLETE: "#DM.Done" + }, + }, + }, + }, + + Done: { + on: { + CLICK: "PromptAndAsk", + }, + }, + }, + }); + + const dmActor = createActor(dmMachine, { + inspect: inspector.inspect, + }).start(); + + dmActor.subscribe((state) => { + /* if you want to log some parts of the state */ + }); + + export function setupButton(element) { + element.addEventListener("click", () => { + dmActor.send({ type: "CLICK" }); + }); + dmActor.getSnapshot().context.ssRef.subscribe((snapshot) => { + element.innerHTML = `${snapshot.value.AsrTtsManager.Ready}`; + }); + } + \ No newline at end of file diff --git a/Code/dm5.js b/Code/dm5.js new file mode 100644 index 0000000..557c0ce --- /dev/null +++ b/Code/dm5.js @@ -0,0 +1,615 @@ +import { and, assign, createActor, setup } from "xstate"; +import { speechstate } from "speechstate"; +import { createBrowserInspector } from "@statelyai/inspect"; +import { KEY } from "./azure.js"; +import { NLU_KEY } from "./azure.js"; +import { sortAndDeduplicateDiagnostics } from "typescript"; +/* +Regarding improveingNLU coverage, I think I need to add more time datas and person datas to the model. +*/ + const inspector = createBrowserInspector(); + const azureLanguageCredentials = { + endpoint: "https://languageresource58888.cognitiveservices.azure.com/language/:analyze-conversations?api-version=2022-10-01-preview" /** your Azure CLU prediction URL */, + key: NLU_KEY /** reference to your Azure CLU key */, + deploymentName: "appointment" /** your Azure CLU deployment */, + projectName: "appointment" /** your Azure CLU project name */, + }; + const azureCredentials = { + endpoint: + "https://northeurope.api.cognitive.microsoft.com/sts/v1.0/issuetoken", + key: KEY, + }; + const settings = { + azureLanguageCredentials: azureLanguageCredentials /** global activation of NLU */, + azureCredentials: azureCredentials, + asrDefaultCompleteTimeout: 0, + asrDefaultNoInputTimeout: 5000, + locale: "en-US", + ttsDefaultVoice: "en-US-DavisNeural", + }; + const grammar_day = { + monday: { day: "Monday" }, + tuesday: { day: "Tuesday" }, + wednesday: { day: "Wednesday" }, + thursday: { day: "Thursday" }, + friday: { day: "Friday" }, + }; + function getThreshold(thre) { + return thre > 0.85; + } + function whoIntent(event) { + return event === "who is adam lambert"; + } + function helpIntent(event) { + return event === "help" || "Help"; + } + function meetIntent(event) { + return event === "create a meeting"; + } + function isInDayGrammar(event) { + return event.toLowerCase() in grammar_day; } + function isInTittleGrammar(event) { + return ((event.toLowerCase() === "virtual meeting")||(event.toLowerCase() === "special meeting")||(event.toLowerCase() === "meeting")); + } + + /* + The ASR threshold is the event.value[0].confidence. So we can use the similar function like getASRThreshold(event.value[0].confidence) + to judge the correction. We can use both of event.nluValue.intents[0].confidenceScore and event.value[0].confidence + as condition for confirmation. For example: + + getASRThreshold(event.value[0].confidence) === true && getThreshold(event.nluValue.intents[0].confidenceScore) === true + to test the model. + + + */ + + const dmMachine = setup({ + actions: { + /* define your actions here */ + say: ({ context }, params) => + context.ssRef.send({ + type: "SPEAK", + value: { + utterance: params, + }, + }), + listen:({ context }) => + context.ssRef.send({ + type: "LISTEN", + value: { nlu: true } /** Local activation of NLU */, + }), + }, + }).createMachine({ + context: { + count: 0, + // for Who is X part + age: undefined, + birthday: undefined, + + // for creating meeting part + weekdate: undefined, + meeting_name: undefined, + + }, + id: "DM", + initial: "Prepare", + states: { + Prepare: { + entry: [ + assign({ + ssRef: ({ spawn }) => spawn(speechstate, { input: settings }), + }), + ({ context }) => context.ssRef.send({ type: "PREPARE" }), + ], + on: { ASRTTS_READY: "WaitToStart" }, + }, + WaitToStart: { + on: { + CLICK: "PromptAndAsk", + }, + }, + PromptAndAsk: { + initial: "Prompt", + states: { + Prompt: { + entry:[{type: "say", params:`Let's start conversation`}], + on: { SPEAK_COMPLETE: "#DM.ProposeQ" }, + }, + }, + }, + /* jump to one of two questions */ + ProposeQ: { + initial: "proposeQ", + states: { + proposeQ: { + entry: [{type: "say", params:`What's your question?`}], + on: { SPEAK_COMPLETE: "Ask" }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + guard: ({ event }) => event.nluValue.topIntent === "None", + target: "#DM.NotQGrammar", + }, + { + guard: (({event}) => meetIntent(event.nluValue.topIntent) && getThreshold (event.nluValue.intents[0].confidenceScore)), + target: "#DM.WhichDate", + }, + { + guard: (({event}) => meetIntent(event.nluValue.topIntent) && !!(getThreshold (event.nluValue.intents[0].confidenceScore))), + target: "#DM.WhichDate", + }, + { + guard: (({event}) => whoIntent(event.nluValue.topIntent) && getThreshold (event.nluValue.intents[0].confidenceScore)), + target: "#DM.WhoisAdam", + }, + { + guard: (({event}) => whoIntent(event.nluValue.topIntent) && !!(getThreshold (event.nluValue.intents[0].confidenceScore))), + target: "#DM.WhoisAdam", + }, + // { + // guard: ({ event }) => event.nluValue.topIntent === "Help", + // target: "#DM.HelpQ", + // }, + "#DM.NotQGrammar", + + ], + ASR_NOINPUT:[{ + guard: ({context}) => context.count >= 3, + target: "#DM.Done", + }, + { + guard: ({context}) => context.count < 3, + actions:({context})=> context.count++, + target: "#DM.NoQVoice", + }], + }, + }, + }, + }, + MakeSureWhichDate:{ + initial: "makeSureWhichDate", + states: { + makeSureWhichDate: { + entry: [{type: "say", params:`Do you say create a meeting?`}], + on: { + SPEAK_COMPLETE: "#DM.WhichDate" + }, + }, + }, + }, + MakeSureWho:{ + initial: "makeSureWho", + states: { + makeSureWho: { + entry: [{type: "say", params:`who is adam lambert`}], + on: { + SPEAK_COMPLETE: "#DM.WhoisAdam" + }, + }, + }, + }, + /* Not received voice */ + NoQVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.ProposeQ" + }, + }, + }, + }, + /* Not in grammar */ + NotQGrammar: { + initial: "notQGrammar", + states: { + notQGrammar: { + entry: [{type: "say", params:`Sorry, I didn't understand. What's your question?, please say "create a meeting" or "who is Adam Lambert"`}], + on: { + SPEAK_COMPLETE: "#DM.ProposeQ" + }, + }, + }, + }, + /* help */ + HelpQ: { + initial: "helpQ", + states: { + helpQ: { + entry: [{type: "say", params:`You need help?`}], + on: { + SPEAK_COMPLETE: "#DM.ProposeQ" + }, + }, + }, + }, + + /* Meeting Time */ + //Answer e.g. Monday to Friday + WhichDate: { + initial: "whichDate", + states: { + whichDate: { + entry: [{type: "say", params:`On which date is the meeting?`}], + on: { + SPEAK_COMPLETE: "Ask" + }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + guard: ({ event }) => isInDayGrammar(event.value[0].utterance) === false, + target: "#DM.NotWhichDateGrammar", + }, + { + guard: ({ event }) => isInDayGrammar(event.nluValue.entities[0].text) === true, + actions: [ + ({ context, event }) => { context.weekdate = event.nluValue.entities[0].text}, + ], + target: "#DM.MeetingTitle", + }, + { + guard: ({ event }) => event.nluValue.entities[0].text === "Help" || "help", + target: "#DM.HelpWhichDate", + }, + + "#DM.NotWhichDateGrammar", + ], + ASR_NOINPUT:[{ + guard: ({context}) => context.count >= 3, + target: "#DM.Done", + }, + { + guard: ({context}) => context.count < 3, + actions:({context}) => context.count++, + target: "#DM.NoWhichDateVoice", + }], + }, + }, + }, + }, + /* Not received voice */ + NoWhichDateVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WhichDate" + }, + }, + }, + }, + /* Not in grammar */ + NotWhichDateGrammar: { + initial: "notWhichDateGrammar", + states: { + notWhichDateGrammar: { + entry: [{type: "say", params:`Sorry, I didn't understand. please say from Monday to Friday`}], + on: { + SPEAK_COMPLETE: "#DM.WhichDate" + }, + }, + }, + }, + /* help */ + HelpWhichDate: { + initial: "helpWhichDate", + states: { + helpWhichDate: { + entry: [{type: "say", params:`You need help?`}], + on: { + SPEAK_COMPLETE: "#DM.WhichDate" + }, + }, + }, + }, + /* Meeting Title */ + //Answer e.g. "special meeting" || "meeting" || "virtual meeting" + MeetingTitle: { + initial: "meetingTitle", + states: { + meetingTitle: { + entry: [{type: "say", params:`What's the meeting title?`}], + on: { + SPEAK_COMPLETE: "Ask" + }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + guard: ({ event }) => isInTittleGrammar(event.value[0].utterance) === false, + target: "#DM.NotMeetingTitleGrammar", + }, + { + //Answer e.g. "special meeting" || "meeting" || "virtual meeting" + guard: ({ event }) => isInTittleGrammar(event.nluValue.entities[0].text) === true, + actions: [ + ({ context, event }) => { context.meeting_name = event.nluValue.entities[0].text}, + ], + target: "#DM.MeetingInfo", + }, + { + guard: ({ event }) => event.nluValue.topIntent == "help" , + target: "#DM.HelpMeetingTitle", + }, + "#DM.NotMeetingTitleGrammar", + ], + ASR_NOINPUT:[{ + guard: ({context}) => context.count >= 3, + target: "#DM.Done", + }, + { + guard: ({context}) => context.count < 3, + actions:({context}) => context.count++, + target: "#DM.NoMeetingTitleVoice", + }], + }, + }, + }, + }, + /* Not received voice */ + NoMeetingTitleVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.MeetingTitle" + }, + }, + }, + }, + /* Not in grammar */ + NotMeetingTitleGrammar: { + initial: "nogrammar", + states: { + nogrammar: { + entry: [{type: "say", params:`Sorry, I didn't understand. please say departmental meeting or meeting or virtual meeting`}], + on: { + SPEAK_COMPLETE: "#DM.MeetingTitle" + }, + }, + }, + }, + /* help */ + HelpMeetingTitle: { + initial: "helpMeetingTitle", + states: { + helpMeetingTitle: { + entry: [{type: "say", params:`You need help?`}], + on: { + SPEAK_COMPLETE: "#DM.MeetingTitle" + }, + }, + }, + }, + /* Repeat the Info of the Meeting */ + MeetingInfo: { + initial: "meetingInfo", + states: { + meetingInfo: { + entry: [{type: "say", params: ({context}) => `The meeting is on ${context.weekdate} and the meeting title is ${context.meeting_name} `}], + on: { + SPEAK_COMPLETE: "#DM.Done" + }, + }, + }, + }, + + /* Who is Adam Lambert */ + //Answer e.g. 42 years old + WhoisAdam: { + initial: "whoisAdam", + states: { + whoisAdam: { + entry: [{type: "say", params:`How old is Adam Lambert?`}], + on: { + SPEAK_COMPLETE: "Ask" + }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + //e.g. Adam Lambert is 42 years old + guard: ({ event }) => whoIntent(event.nluValue.topIntent) === false , + target: "#DM.NotWhoGrammar", + }, + { + //e.g. 42 years old + guard: ({ event }) => event.nluValue.entities[0].text === "42 years old", + actions: [ + ({ context, event }) => { context.age = event.nluValue.entities[0].text}, + ], + target: "#DM.WhatAdamBirth", + }, + { + guard: ({ event }) => event.nluValue.topIntent == "help" , + target: "#DM.HelpWho", + }, + "#DM.NotWhoGrammar" + ], + ASR_NOINPUT:[{ + guard: ({context}) => context.count >= 3, + target: "#DM.Done", + }, + { + guard: ({context}) => context.count < 3, + actions:({context}) => context.count++, + target: "#DM.NoAdamVoice", + }], + }, + }, + }, + }, + /* Not received voice */ + NoAdamVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WhoisAdam" + }, + }, + }, + }, + /* Not in grammar */ + NotWhoGrammar: { + initial: "notWhoGrammar", + states: { + notWhoGrammar: { + entry: [{type: "say", params:`Sorry, I didn't understand. please say Adam Lambert is 42 years old`}], + on: { + SPEAK_COMPLETE: "#DM.WhoisAdam" + }, + }, + }, + }, + /* help */ + HelpWho: { + initial: "helpWho", + states: { + helpWho: { + entry: [{type: "say", params:`You need help?`}], + on: { + SPEAK_COMPLETE: "#DM.WhoisAdam" + }, + }, + }, + }, + /* When is Adam Lambert's birthday */ + //Answer e.g. Adam Lambert is born in Jan 29th + WhatAdamBirth: { + initial: "whatAdamBirth", + states: { + whatAdamBirth: { + entry: [{type: "say", params:`When is Adam Lambert's birthday?`}], + on: { + SPEAK_COMPLETE: "Ask" + }, + }, + Ask: { + entry: "listen", + on: { + RECOGNISED: + [ + { + //e.g. Adam Lambert is born in Jan 29th + guard: ({ event }) => whoIntent(event.nluValue.topIntent) ===false , + target: "#DM.NotAdamBirthDayGrammar", + }, + { + //e.g. Adam Lambert is born in Jan 29th + guard: ({ event }) => whoIntent(event.nluValue.topIntent) === true , + actions: [ + ({ context, event }) => { context.birthday = event.nluValue.entities[0].text}, + ], + target: "#DM.AdamInfo", + }, + { + guard: ({ event }) => event.nluValue.topIntent === "help" , + target: "#DM.HelpAdamBirthDay", + }, + "#DM.NotAdamBirthDayGrammar" + ], + ASR_NOINPUT:[{ + guard: ({context}) => context.count >= 3, + target: "#DM.Done", + }, + { + guard: ({context}) => context.count < 3, + actions:({context}) => context.count++, + target: "#DM.NoAdamBirthDayVoice", + }], + }, + }, + }, + }, + /* Not received voice */ + NoAdamBirthDayVoice: { + initial: "Novoice", + states: { + Novoice: { + entry: [{type: "say", params:`I didn’t hear you`}], + on: { + SPEAK_COMPLETE: "#DM.WhatAdamBirth" + }, + }, + }, + }, + /* Not in grammar */ + NotAdamBirthDayGrammar: { + initial: "notAdamBirthDayGrammar", + states: { + notAdamBirthDayGrammar: { + entry: [{type: "say", params:`Sorry, I didn't understand. please say Adam Lambert is born in Jan 29th`}], + on: { + SPEAK_COMPLETE: "#DM.WhatAdamBirth" + }, + }, + }, + }, + /* help */ + HelpAdamBirthDay: { + initial: "helpAdamBirthDay", + states: { + helpAdamBirthDay: { + entry: [{type: "say", params:`You need help?`}], + on: { + SPEAK_COMPLETE: "#DM.WhatAdamBirth" + }, + }, + }, + }, + /* Repeat the Info of the Person */ + AdamInfo: { + initial: "adamInfo", + states: { + adamInfo: { + entry: [{type: "say", params: ({context}) => `Adam Lambert is ${context.age} and his birthday is ${context.birthday} `}], + on: { + SPEAK_COMPLETE: "#DM.Done" + }, + }, + }, + }, + Done: { + on: { + CLICK: "PromptAndAsk", + }, + }, + }, + }); + + const dmActor = createActor(dmMachine, { + inspect: inspector.inspect, + }).start(); + + dmActor.subscribe((state) => { + /* if you want to log some parts of the state */ + }); + + export function setupButton(element) { + element.addEventListener("click", () => { + dmActor.send({ type: "CLICK" }); + }); + dmActor.getSnapshot().context.ssRef.subscribe((snapshot) => { + element.innerHTML = `${snapshot.value.AsrTtsManager.Ready}`; + }); + } + \ No newline at end of file diff --git a/Code/lab3.mp3 b/Code/lab3.mp3 new file mode 100644 index 0000000..cf45e30 Binary files /dev/null and b/Code/lab3.mp3 differ diff --git a/Code/lab3.txt b/Code/lab3.txt new file mode 100644 index 0000000..11e2a58 --- /dev/null +++ b/Code/lab3.txt @@ -0,0 +1,19 @@ +Original performance: https://www.youtube.com/watch?v=VjMoRb5f-EQ + + + + + +I celebrate myself, and sing myself,  +And what I assume you shall assume, +For every atom belonging to me as good belongs to you. +I loafe and invite my soul, +I lean and loafe at my ease observing a spear of summer grass. +My tongue, every atom of my blood, form’d from this soil, this air, +Born here of parents born here from parents the same, and their parents the same, +I, now thirty-seven years old in perfect health begin, +Hoping to cease not till death. +Creeds and schools in abeyance, +Retiring back a while sufficed at what they are, but never forgotten, +I harbor for good or bad, I permit to speak at every hazard, +Nature without check with original energy. \ No newline at end of file diff --git a/Code/main.js b/Code/main.js index 7b1df80..5d0270a 100644 --- a/Code/main.js +++ b/Code/main.js @@ -1,5 +1,5 @@ import "./style.css"; -import { setupButton } from "./dm4.js"; +import { setupButton } from "./dm5.js"; document.querySelector("#app").innerHTML = `
diff --git a/Code/package.json b/Code/package.json index 815868a..67512fb 100644 --- a/Code/package.json +++ b/Code/package.json @@ -15,7 +15,7 @@ }, "dependencies": { "@statelyai/inspect": "^0.2.2", - "speechstate": "^2.0.0-beta.6", + "speechstate": "^2.0.0-beta.7", "xstate": "^5.5.1" } -} +} \ No newline at end of file diff --git a/Code/yarn.lock b/Code/yarn.lock index 8f292ac..8e3b7c1 100644 --- a/Code/yarn.lock +++ b/Code/yarn.lock @@ -2950,7 +2950,7 @@ __metadata: resolution: "js-vite@workspace:." dependencies: "@statelyai/inspect": ^0.2.2 - speechstate: ^2.0.0-beta.6 + speechstate: ^2.0.0-beta.7 typescript: ^5.3.3 typescript-language-server: ^4.3.1 vite: ^5.0.8 @@ -3789,15 +3789,15 @@ __metadata: languageName: node linkType: hard -"speechstate@npm:^2.0.0-beta.6": - version: 2.0.0-beta.6 - resolution: "speechstate@npm:2.0.0-beta.6" +"speechstate@npm:^2.0.0-beta.7": + version: 2.0.0-beta.7 + resolution: "speechstate@npm:2.0.0-beta.7" dependencies: "@statelyai/inspect": ^0.2.3 microsoft-cognitiveservices-speech-sdk: ^1.31.0 web-speech-cognitive-services: ^7.1.3 xstate: ^5.5.1 - checksum: 8d7f544cfcf44a0588f297ef9bfb2071ac41be88e5334c7b809cf1d552331a1db97a7d60e152d763ab4041730f5c3b08c621e1045782eba7b3a43f2d405a2012 + checksum: f74309d2a842f8db18e7e537b3fbd36a9583e50fcfc38a56aef0086f0334e451df2b239a62c95d7e06d940b48cc3347c4df790163f3221f03f1004c48e43d807 languageName: node linkType: hard @@ -4205,4 +4205,4 @@ __metadata: resolution: "yocto-queue@npm:1.0.0" checksum: 2cac84540f65c64ccc1683c267edce396b26b1e931aa429660aefac8fbe0188167b7aee815a3c22fa59a28a58d898d1a2b1825048f834d8d629f4c2a5d443801 languageName: node - linkType: hard + linkType: hard \ No newline at end of file diff --git a/CustomSpeechDatasets/PlainText.txt b/CustomSpeechDatasets/PlainText.txt new file mode 100644 index 0000000..8f66d25 --- /dev/null +++ b/CustomSpeechDatasets/PlainText.txt @@ -0,0 +1,4 @@ +kranichfeld +fredmund +asad +riyad diff --git a/CustomSpeechDatasets/PronunciationDatasets.txt b/CustomSpeechDatasets/PronunciationDatasets.txt new file mode 100644 index 0000000..83bdefb --- /dev/null +++ b/CustomSpeechDatasets/PronunciationDatasets.txt @@ -0,0 +1,4 @@ +Fredmund fred mond fredmund +Kranichfeld cry nik feld kranichfeld +Riyad ray yard riyad +Asad a zad asad diff --git a/CustomSpeechLog/Names.json b/CustomSpeechLog/Names.json new file mode 100644 index 0000000..538478f --- /dev/null +++ b/CustomSpeechLog/Names.json @@ -0,0 +1 @@ +{"source":"online","requestId":"f93f77d0-14d7-48a4-ae0e-5cf0221c5d63","timestamp":"2024-02-26T01:06:55.0508049Z","durationTicks":424960000,"duration":"PT42.496S","combinedRecognizedPhrases":[{"channel":0,"lexical":"hello world Kranichfeld Kranichfeld Kranichfeld chronic failed Fredmund Fredmund Fredmund Fredmund Asad Asad Asad Riyad riyad Riyad","itn":"hello world Kranichfeld Kranichfeld Kranichfeld chronic failed Fredmund Fredmund Fredmund Fredmund Asad Asad Asad Riyad riyad Riyad","maskedItn":"hello world kranichfeld kranichfeld kranichfeld chronic failed fredmund fredmund fredmund fredmund asad asad asad riyad riyad riyad","display":"Hello world. Kranichfeld. Kranichfeld Kranichfeld, Chronic failed. Fredmund. Fredmund. Fredmund. Fredmund. Asad. Asad. Asad. Riyad. Riyad. Riyad."}],"recognizedPhrases":[{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":8500000,"offset":"PT0.85S","durationInTicks":12700000,"duration":"PT1.27S","nBest":[{"confidence":0.6928716897964478,"lexical":"hello world","itn":"hello world","maskedItn":"hello world","display":"Hello world.","words":[{"confidence":0.42071330547332764,"lexical":"hello","offsetInTicks":8500000,"offset":"PT0.85S","durationInTicks":4700000,"duration":"PT0.47S"},{"confidence":0.8561667203903198,"lexical":"world","offsetInTicks":13300000,"offset":"PT1.33S","durationInTicks":7900000,"duration":"PT0.79S"}]},{"confidence":0.529560923576355,"lexical":"halo world","itn":"halo world","maskedItn":"halo world","display":"halo world","words":[{"confidence":0.21287989616394043,"lexical":"halo","offsetInTicks":8600000,"offset":"PT0.86S","durationInTicks":4600000,"duration":"PT0.46S"},{"confidence":0.7156110405921936,"lexical":"world","offsetInTicks":13300000,"offset":"PT1.33S","durationInTicks":7900000,"duration":"PT0.79S"}]},{"confidence":0.441134512424469,"lexical":"hello word","itn":"hello word","maskedItn":"hello word","display":"hello word","words":[{"confidence":0.42071330547332764,"lexical":"hello","offsetInTicks":8500000,"offset":"PT0.85S","durationInTicks":4700000,"duration":"PT0.47S"},{"confidence":0.45338723063468933,"lexical":"word","offsetInTicks":13300000,"offset":"PT1.33S","durationInTicks":7900000,"duration":"PT0.79S"}]},{"confidence":0.4524916112422943,"lexical":"hilo world","itn":"hilo world","maskedItn":"hilo world","display":"hilo world","words":[{"confidence":0.20806163549423218,"lexical":"hilo","offsetInTicks":8500000,"offset":"PT0.85S","durationInTicks":4700000,"duration":"PT0.47S"},{"confidence":0.5991495847702026,"lexical":"world","offsetInTicks":13300000,"offset":"PT1.33S","durationInTicks":7900000,"duration":"PT0.79S"}]},{"confidence":0.4904349744319916,"lexical":"hitler world","itn":"hitler world","maskedItn":"hitler world","display":"hitler world","words":[{"confidence":0.08951228857040405,"lexical":"hitler","offsetInTicks":8500000,"offset":"PT0.85S","durationInTicks":4500000,"duration":"PT0.45S"},{"confidence":0.7153428196907043,"lexical":"world","offsetInTicks":13100000,"offset":"PT1.31S","durationInTicks":8100000,"duration":"PT0.81S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":62900000,"offset":"PT6.29S","durationInTicks":16700000,"duration":"PT1.67S","nBest":[{"confidence":0.919089674949646,"lexical":"Kranichfeld","itn":"Kranichfeld","maskedItn":"kranichfeld","display":"Kranichfeld.","words":[{"confidence":0.919089674949646,"lexical":"Kranichfeld","offsetInTicks":62900000,"offset":"PT6.29S","durationInTicks":16700000,"duration":"PT1.67S"}]},{"confidence":0.4794572591781616,"lexical":"koranic field","itn":"koranic field","maskedItn":"koranic field","display":"koranic field","words":[{"confidence":0.2196723222732544,"lexical":"koranic","offsetInTicks":62800000,"offset":"PT6.28S","durationInTicks":9400000,"duration":"PT0.94S"},{"confidence":0.812964916229248,"lexical":"field","offsetInTicks":72300000,"offset":"PT7.23S","durationInTicks":7300000,"duration":"PT0.73S"}]},{"confidence":0.4615706205368042,"lexical":"chronic field","itn":"chronic field","maskedItn":"chronic field","display":"chronic field","words":[{"confidence":0.33352547883987427,"lexical":"chronic","offsetInTicks":63100000,"offset":"PT6.31S","durationInTicks":9100000,"duration":"PT0.91S"},{"confidence":0.6207618713378906,"lexical":"field","offsetInTicks":72300000,"offset":"PT7.23S","durationInTicks":7300000,"duration":"PT0.73S"}]},{"confidence":0.46764400601387024,"lexical":"quranic field","itn":"quranic field","maskedItn":"quranic field","display":"quranic field","words":[{"confidence":0.18872392177581787,"lexical":"quranic","offsetInTicks":63100000,"offset":"PT6.31S","durationInTicks":9100000,"duration":"PT0.91S"},{"confidence":0.8144094944000244,"lexical":"field","offsetInTicks":72300000,"offset":"PT7.23S","durationInTicks":7300000,"duration":"PT0.73S"}]},{"confidence":0.4687955677509308,"lexical":"kronic field","itn":"kronic field","maskedItn":"kronic field","display":"kronic field","words":[{"confidence":0.18482959270477295,"lexical":"kronic","offsetInTicks":62800000,"offset":"PT6.28S","durationInTicks":9400000,"duration":"PT0.94S"},{"confidence":0.8333464860916138,"lexical":"field","offsetInTicks":72300000,"offset":"PT7.23S","durationInTicks":7300000,"duration":"PT0.73S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":88800000,"offset":"PT8.88S","durationInTicks":55400000,"duration":"PT5.54S","nBest":[{"confidence":0.6299960613250732,"lexical":"Kranichfeld Kranichfeld chronic failed","itn":"Kranichfeld Kranichfeld chronic failed","maskedItn":"kranichfeld kranichfeld chronic failed","display":"Kranichfeld Kranichfeld, Chronic failed.","words":[{"confidence":0.8846613168716431,"lexical":"Kranichfeld","offsetInTicks":88800000,"offset":"PT8.88S","durationInTicks":16500000,"duration":"PT1.65S"},{"confidence":0.5554677248001099,"lexical":"Kranichfeld","offsetInTicks":110400000,"offset":"PT11.04S","durationInTicks":18100000,"duration":"PT1.81S"},{"confidence":0.3940911889076233,"lexical":"chronic","offsetInTicks":129400000,"offset":"PT12.94S","durationInTicks":7500000,"duration":"PT0.75S"},{"confidence":0.4823044538497925,"lexical":"failed","offsetInTicks":137000000,"offset":"PT13.7S","durationInTicks":7200000,"duration":"PT0.72S"}]},{"confidence":0.5571924448013306,"lexical":"kranichfeld koranic field chronic failed","itn":"kranichfeld koranic field chronic failed","maskedItn":"kranichfeld koranic field chronic failed","display":"Kranichfeld koranic field chronic failed","words":[{"confidence":0.8846613168716431,"lexical":"Kranichfeld","offsetInTicks":88800000,"offset":"PT8.88S","durationInTicks":16500000,"duration":"PT1.65S"},{"confidence":0.29086172580718994,"lexical":"koranic","offsetInTicks":110400000,"offset":"PT11.04S","durationInTicks":7900000,"duration":"PT0.79S"},{"confidence":0.4727417230606079,"lexical":"field","offsetInTicks":118400000,"offset":"PT11.84S","durationInTicks":10100000,"duration":"PT1.01S"},{"confidence":0.30755287408828735,"lexical":"chronic","offsetInTicks":129400000,"offset":"PT12.94S","durationInTicks":7500000,"duration":"PT0.75S"},{"confidence":0.4823044538497925,"lexical":"failed","offsetInTicks":137000000,"offset":"PT13.7S","durationInTicks":7200000,"duration":"PT0.72S"}]},{"confidence":0.567085325717926,"lexical":"kranichfeld koranic failed chronic failed","itn":"kranichfeld koranic failed chronic failed","maskedItn":"kranichfeld koranic failed chronic failed","display":"Kranichfeld koranic failed chronic failed","words":[{"confidence":0.8846613168716431,"lexical":"Kranichfeld","offsetInTicks":88800000,"offset":"PT8.88S","durationInTicks":16500000,"duration":"PT1.65S"},{"confidence":0.29086172580718994,"lexical":"koranic","offsetInTicks":110400000,"offset":"PT11.04S","durationInTicks":7900000,"duration":"PT0.79S"},{"confidence":0.5113087296485901,"lexical":"failed","offsetInTicks":118400000,"offset":"PT11.84S","durationInTicks":9100000,"duration":"PT0.91S"},{"confidence":0.31314751505851746,"lexical":"chronic","offsetInTicks":129400000,"offset":"PT12.94S","durationInTicks":7500000,"duration":"PT0.75S"},{"confidence":0.4823044538497925,"lexical":"failed","offsetInTicks":137000000,"offset":"PT13.7S","durationInTicks":7200000,"duration":"PT0.72S"}]},{"confidence":0.6198424100875854,"lexical":"kranichfeld kranichfeld chronic field","itn":"kranichfeld kranichfeld chronic field","maskedItn":"kranichfeld kranichfeld chronic field","display":"Kranichfeld Kranichfeld chronic field","words":[{"confidence":0.8846613168716431,"lexical":"Kranichfeld","offsetInTicks":88800000,"offset":"PT8.88S","durationInTicks":16500000,"duration":"PT1.65S"},{"confidence":0.5554677248001099,"lexical":"Kranichfeld","offsetInTicks":110400000,"offset":"PT11.04S","durationInTicks":18100000,"duration":"PT1.81S"},{"confidence":0.3940911889076233,"lexical":"chronic","offsetInTicks":129400000,"offset":"PT12.94S","durationInTicks":7500000,"duration":"PT0.75S"},{"confidence":0.4131758213043213,"lexical":"field","offsetInTicks":137000000,"offset":"PT13.7S","durationInTicks":7200000,"duration":"PT0.72S"}]},{"confidence":0.6114646196365356,"lexical":"kranichfeld kranichfeld koranic failed","itn":"kranichfeld kranichfeld koranic failed","maskedItn":"kranichfeld kranichfeld koranic failed","display":"Kranichfeld Kranichfeld koranic failed","words":[{"confidence":0.8846613168716431,"lexical":"Kranichfeld","offsetInTicks":88800000,"offset":"PT8.88S","durationInTicks":16500000,"duration":"PT1.65S"},{"confidence":0.5554677248001099,"lexical":"Kranichfeld","offsetInTicks":110400000,"offset":"PT11.04S","durationInTicks":18100000,"duration":"PT1.81S"},{"confidence":0.18735221028327942,"lexical":"koranic","offsetInTicks":129200000,"offset":"PT12.92S","durationInTicks":7700000,"duration":"PT0.77S"},{"confidence":0.5829928517341614,"lexical":"failed","offsetInTicks":137000000,"offset":"PT13.7S","durationInTicks":7200000,"duration":"PT0.72S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":155600000,"offset":"PT15.56S","durationInTicks":45700000,"duration":"PT4.57S","nBest":[{"confidence":0.9581303596496582,"lexical":"Fredmund Fredmund Fredmund","itn":"Fredmund Fredmund Fredmund","maskedItn":"fredmund fredmund fredmund","display":"Fredmund. Fredmund. Fredmund.","words":[{"confidence":0.9439952373504639,"lexical":"Fredmund","offsetInTicks":155600000,"offset":"PT15.56S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.9826977252960205,"lexical":"Fredmund","offsetInTicks":170400000,"offset":"PT17.04S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.9475401639938354,"lexical":"Fredmund","offsetInTicks":188200000,"offset":"PT18.82S","durationInTicks":13100000,"duration":"PT1.31S"}]},{"confidence":0.8433017730712891,"lexical":"fredmund fredmund fried mound","itn":"fredmund fredmund fried mound","maskedItn":"fredmund fredmund fried mound","display":"Fredmund Fredmund fried mound","words":[{"confidence":0.9439952373504639,"lexical":"Fredmund","offsetInTicks":155600000,"offset":"PT15.56S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.9826977252960205,"lexical":"Fredmund","offsetInTicks":170400000,"offset":"PT17.04S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.6995341181755066,"lexical":"fried","offsetInTicks":188200000,"offset":"PT18.82S","durationInTicks":5500000,"duration":"PT0.55S"},{"confidence":0.5259202718734741,"lexical":"mound","offsetInTicks":193800000,"offset":"PT19.38S","durationInTicks":7500000,"duration":"PT0.75S"}]},{"confidence":0.8082222938537598,"lexical":"fredmund fredmund fried mund","itn":"fredmund fredmund fried mund","maskedItn":"fredmund fredmund fried mund","display":"Fredmund Fredmund fried mund","words":[{"confidence":0.9439952373504639,"lexical":"Fredmund","offsetInTicks":155600000,"offset":"PT15.56S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.9826977252960205,"lexical":"Fredmund","offsetInTicks":170400000,"offset":"PT17.04S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.6974375247955322,"lexical":"fried","offsetInTicks":188200000,"offset":"PT18.82S","durationInTicks":5300000,"duration":"PT0.53S"},{"confidence":0.35192885994911194,"lexical":"mund","offsetInTicks":193600000,"offset":"PT19.36S","durationInTicks":7700000,"duration":"PT0.77S"}]},{"confidence":0.8638592958450317,"lexical":"fredmund fredmund 's fredmund","itn":"fredmund fredmund 's fredmund","maskedItn":"fredmund fredmund 's fredmund","display":"Fredmund Fredmund 's Fredmund","words":[{"confidence":0.9439952373504639,"lexical":"Fredmund","offsetInTicks":155600000,"offset":"PT15.56S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.9826977252960205,"lexical":"Fredmund","offsetInTicks":170400000,"offset":"PT17.04S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.007642924785614014,"lexical":"'s","offsetInTicks":183800000,"offset":"PT18.38S","durationInTicks":4300000,"duration":"PT0.43S"},{"confidence":0.9472756385803223,"lexical":"Fredmund","offsetInTicks":188200000,"offset":"PT18.82S","durationInTicks":13100000,"duration":"PT1.31S"}]},{"confidence":0.9275484085083008,"lexical":"fredmund 's fredmund fredmund","itn":"fredmund 's fredmund fredmund","maskedItn":"fredmund 's fredmund fredmund","display":"Fredmund 's Fredmund Fredmund","words":[{"confidence":0.9439952373504639,"lexical":"Fredmund","offsetInTicks":155600000,"offset":"PT15.56S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.05512291193008423,"lexical":"'s","offsetInTicks":169000000,"offset":"PT16.9S","durationInTicks":1300000,"duration":"PT0.13S"},{"confidence":0.982556939125061,"lexical":"Fredmund","offsetInTicks":170400000,"offset":"PT17.04S","durationInTicks":13300000,"duration":"PT1.33S"},{"confidence":0.9475401639938354,"lexical":"Fredmund","offsetInTicks":188200000,"offset":"PT18.82S","durationInTicks":13100000,"duration":"PT1.31S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":212300000,"offset":"PT21.23S","durationInTicks":13100000,"duration":"PT1.31S","nBest":[{"confidence":0.815838634967804,"lexical":"Fredmund","itn":"Fredmund","maskedItn":"fredmund","display":"Fredmund.","words":[{"confidence":0.815838634967804,"lexical":"Fredmund","offsetInTicks":212300000,"offset":"PT21.23S","durationInTicks":13100000,"duration":"PT1.31S"}]},{"confidence":0.38774561882019043,"lexical":"friedman","itn":"friedman","maskedItn":"friedman","display":"friedman","words":[{"confidence":0.38774561882019043,"lexical":"friedman","offsetInTicks":212300000,"offset":"PT21.23S","durationInTicks":13100000,"duration":"PT1.31S"}]},{"confidence":0.44542425870895386,"lexical":"friedman and","itn":"friedman and","maskedItn":"friedman and","display":"friedman and","words":[{"confidence":0.30463021993637085,"lexical":"friedman","offsetInTicks":212300000,"offset":"PT21.23S","durationInTicks":9200000,"duration":"PT0.92S"},{"confidence":0.7811639308929443,"lexical":"and","offsetInTicks":221600000,"offset":"PT22.16S","durationInTicks":3800000,"duration":"PT0.38S"}]},{"confidence":0.27841654419898987,"lexical":"fredmund and","itn":"fredmund and","maskedItn":"fredmund and","display":"Fredmund and","words":[{"confidence":0.1548026204109192,"lexical":"Fredmund","offsetInTicks":212300000,"offset":"PT21.23S","durationInTicks":9600000,"duration":"PT0.96S"},{"confidence":0.6210036873817444,"lexical":"and","offsetInTicks":222000000,"offset":"PT22.2S","durationInTicks":3400000,"duration":"PT0.34S"}]},{"confidence":0.20437006652355194,"lexical":"friedmann","itn":"friedmann","maskedItn":"friedmann","display":"friedmann","words":[{"confidence":0.20437008142471313,"lexical":"friedmann","offsetInTicks":212300000,"offset":"PT21.23S","durationInTicks":13100000,"duration":"PT1.31S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":247800000,"offset":"PT24.78S","durationInTicks":10100000,"duration":"PT1.01S","nBest":[{"confidence":0.9741714000701904,"lexical":"Asad","itn":"Asad","maskedItn":"asad","display":"Asad.","words":[{"confidence":0.9741714000701904,"lexical":"Asad","offsetInTicks":247800000,"offset":"PT24.78S","durationInTicks":10100000,"duration":"PT1.01S"}]},{"confidence":0.9428275227546692,"lexical":"i asad","itn":"i asad","maskedItn":"i asad","display":"i Asad","words":[{"confidence":0.5177058577537537,"lexical":"i","offsetInTicks":248100000,"offset":"PT24.81S","durationInTicks":400000,"duration":"PT0.04S"},{"confidence":0.9654403328895569,"lexical":"Asad","offsetInTicks":248600000,"offset":"PT24.86S","durationInTicks":9300000,"duration":"PT0.93S"}]},{"confidence":0.8654002547264099,"lexical":"as asad","itn":"as asad","maskedItn":"as asad","display":"as Asad","words":[{"confidence":0.6741927862167358,"lexical":"as","offsetInTicks":247800000,"offset":"PT24.78S","durationInTicks":2500000,"duration":"PT0.25S"},{"confidence":0.9308133125305176,"lexical":"Asad","offsetInTicks":250400000,"offset":"PT25.04S","durationInTicks":7500000,"duration":"PT0.75S"}]},{"confidence":0.9204444289207458,"lexical":"a asad","itn":"a asad","maskedItn":"a asad","display":"a Asad","words":[{"confidence":0.463481605052948,"lexical":"a","offsetInTicks":247800000,"offset":"PT24.78S","durationInTicks":700000,"duration":"PT0.07S"},{"confidence":0.9593348503112793,"lexical":"Asad","offsetInTicks":248600000,"offset":"PT24.86S","durationInTicks":9300000,"duration":"PT0.93S"}]},{"confidence":0.8034237027168274,"lexical":"asad asad","itn":"asad asad","maskedItn":"asad asad","display":"Asad Asad","words":[{"confidence":0.7456552982330322,"lexical":"Asad","offsetInTicks":247800000,"offset":"PT24.78S","durationInTicks":5600000,"duration":"PT0.56S"},{"confidence":0.8765969276428223,"lexical":"Asad","offsetInTicks":253500000,"offset":"PT25.35S","durationInTicks":4400000,"duration":"PT0.44S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":268800000,"offset":"PT26.88S","durationInTicks":10800000,"duration":"PT1.08S","nBest":[{"confidence":0.9655779600143433,"lexical":"Asad","itn":"Asad","maskedItn":"asad","display":"Asad.","words":[{"confidence":0.9655779600143433,"lexical":"Asad","offsetInTicks":268800000,"offset":"PT26.88S","durationInTicks":10800000,"duration":"PT1.08S"}]},{"confidence":0.8819063901901245,"lexical":"at asad","itn":"at asad","maskedItn":"at asad","display":"at Asad","words":[{"confidence":0.5534899830818176,"lexical":"at","offsetInTicks":268700000,"offset":"PT26.87S","durationInTicks":2100000,"duration":"PT0.21S"},{"confidence":0.96401047706604,"lexical":"Asad","offsetInTicks":270900000,"offset":"PT27.09S","durationInTicks":8700000,"duration":"PT0.87S"}]},{"confidence":0.8662745952606201,"lexical":"add asad","itn":"add asad","maskedItn":"add asad","display":"add Asad","words":[{"confidence":0.500739336013794,"lexical":"add","offsetInTicks":268700000,"offset":"PT26.87S","durationInTicks":2200000,"duration":"PT0.22S"},{"confidence":0.9629103541374207,"lexical":"Asad","offsetInTicks":271000000,"offset":"PT27.1S","durationInTicks":8600000,"duration":"PT0.86S"}]},{"confidence":0.919380784034729,"lexical":"i asad","itn":"i asad","maskedItn":"i asad","display":"i Asad","words":[{"confidence":0.5126737356185913,"lexical":"i","offsetInTicks":268800000,"offset":"PT26.88S","durationInTicks":500000,"duration":"PT0.05S"},{"confidence":0.9430724382400513,"lexical":"Asad","offsetInTicks":269400000,"offset":"PT26.94S","durationInTicks":10200000,"duration":"PT1.02S"}]},{"confidence":0.8482287526130676,"lexical":"asad asad","itn":"asad asad","maskedItn":"asad asad","display":"Asad Asad","words":[{"confidence":0.3569096326828003,"lexical":"Asad","offsetInTicks":268800000,"offset":"PT26.88S","durationInTicks":2000000,"duration":"PT0.2S"},{"confidence":0.96547532081604,"lexical":"Asad","offsetInTicks":270900000,"offset":"PT27.09S","durationInTicks":8700000,"duration":"PT0.87S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":296100000,"offset":"PT29.61S","durationInTicks":10700000,"duration":"PT1.07S","nBest":[{"confidence":0.7346482872962952,"lexical":"Asad","itn":"Asad","maskedItn":"asad","display":"Asad.","words":[{"confidence":0.7346482872962952,"lexical":"Asad","offsetInTicks":296100000,"offset":"PT29.61S","durationInTicks":10700000,"duration":"PT1.07S"}]},{"confidence":0.6704127788543701,"lexical":"at asad","itn":"at asad","maskedItn":"at asad","display":"at Asad","words":[{"confidence":0.500914990901947,"lexical":"at","offsetInTicks":296000000,"offset":"PT29.6S","durationInTicks":2400000,"duration":"PT0.24S"},{"confidence":0.7208585739135742,"lexical":"Asad","offsetInTicks":298500000,"offset":"PT29.85S","durationInTicks":8300000,"duration":"PT0.83S"}]},{"confidence":0.6315980553627014,"lexical":"add asad","itn":"add asad","maskedItn":"add asad","display":"add Asad","words":[{"confidence":0.33433499932289124,"lexical":"add","offsetInTicks":296000000,"offset":"PT29.6S","durationInTicks":2300000,"duration":"PT0.23S"},{"confidence":0.7155311703681946,"lexical":"Asad","offsetInTicks":298400000,"offset":"PT29.84S","durationInTicks":8400000,"duration":"PT0.84S"}]},{"confidence":0.6468538641929626,"lexical":"asad asad","itn":"asad asad","maskedItn":"asad asad","display":"Asad Asad","words":[{"confidence":0.3309154212474823,"lexical":"Asad","offsetInTicks":296100000,"offset":"PT29.61S","durationInTicks":2200000,"duration":"PT0.22S"},{"confidence":0.7323430776596069,"lexical":"Asad","offsetInTicks":298400000,"offset":"PT29.84S","durationInTicks":8400000,"duration":"PT0.84S"}]},{"confidence":0.45493072271347046,"lexical":"that's asad","itn":"that's asad","maskedItn":"that's asad","display":"that's Asad","words":[{"confidence":0.27926161885261536,"lexical":"that's","offsetInTicks":296100000,"offset":"PT29.61S","durationInTicks":3500000,"duration":"PT0.35S"},{"confidence":0.5427652597427368,"lexical":"Asad","offsetInTicks":299700000,"offset":"PT29.97S","durationInTicks":7100000,"duration":"PT0.71S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":332200000,"offset":"PT33.22S","durationInTicks":12900000,"duration":"PT1.29S","nBest":[{"confidence":0.6508388519287109,"lexical":"Riyad","itn":"Riyad","maskedItn":"riyad","display":"Riyad.","words":[{"confidence":0.6508388519287109,"lexical":"Riyad","offsetInTicks":332200000,"offset":"PT33.22S","durationInTicks":12900000,"duration":"PT1.29S"}]},{"confidence":0.31514716148376465,"lexical":"riyadh","itn":"riyadh","maskedItn":"riyadh","display":"riyadh","words":[{"confidence":0.31514716148376465,"lexical":"riyadh","offsetInTicks":332200000,"offset":"PT33.22S","durationInTicks":12900000,"duration":"PT1.29S"}]},{"confidence":0.24935093522071838,"lexical":"riad","itn":"riad","maskedItn":"riad","display":"riad","words":[{"confidence":0.24935093522071838,"lexical":"riad","offsetInTicks":332200000,"offset":"PT33.22S","durationInTicks":12900000,"duration":"PT1.29S"}]},{"confidence":0.2568085193634033,"lexical":"riyad","itn":"riyad","maskedItn":"riyad","display":"riyad","words":[{"confidence":0.2568085193634033,"lexical":"riyad","offsetInTicks":332200000,"offset":"PT33.22S","durationInTicks":12900000,"duration":"PT1.29S"}]},{"confidence":0.21613945066928864,"lexical":"ryad","itn":"ryad","maskedItn":"ryad","display":"ryad","words":[{"confidence":0.21613946557044983,"lexical":"ryad","offsetInTicks":332200000,"offset":"PT33.22S","durationInTicks":12900000,"duration":"PT1.29S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":353000000,"offset":"PT35.3S","durationInTicks":12600000,"duration":"PT1.26S","nBest":[{"confidence":0.31582558155059814,"lexical":"riyad","itn":"riyad","maskedItn":"riyad","display":"Riyad.","words":[{"confidence":0.31582558155059814,"lexical":"riyad","offsetInTicks":353000000,"offset":"PT35.3S","durationInTicks":12600000,"duration":"PT1.26S"}]},{"confidence":0.28669655323028564,"lexical":"riad","itn":"riad","maskedItn":"riad","display":"riad","words":[{"confidence":0.28669655323028564,"lexical":"riad","offsetInTicks":353000000,"offset":"PT35.3S","durationInTicks":12600000,"duration":"PT1.26S"}]},{"confidence":0.26221299171447754,"lexical":"ryad","itn":"ryad","maskedItn":"ryad","display":"ryad","words":[{"confidence":0.26221299171447754,"lexical":"ryad","offsetInTicks":353000000,"offset":"PT35.3S","durationInTicks":12600000,"duration":"PT1.26S"}]},{"confidence":0.3625088930130005,"lexical":"riyadh","itn":"riyadh","maskedItn":"riyadh","display":"riyadh","words":[{"confidence":0.3625088930130005,"lexical":"riyadh","offsetInTicks":353000000,"offset":"PT35.3S","durationInTicks":12600000,"duration":"PT1.26S"}]},{"confidence":0.694011926651001,"lexical":"riyad","itn":"riyad","maskedItn":"riyad","display":"Riyad","words":[{"confidence":0.694011926651001,"lexical":"Riyad","offsetInTicks":353000000,"offset":"PT35.3S","durationInTicks":12600000,"duration":"PT1.26S"}]}]},{"recognitionStatus":"Success","channel":0,"speaker":0,"offsetInTicks":377300000,"offset":"PT37.73S","durationInTicks":13500000,"duration":"PT1.35S","nBest":[{"confidence":0.7539938688278198,"lexical":"Riyad","itn":"Riyad","maskedItn":"riyad","display":"Riyad.","words":[{"confidence":0.753993809223175,"lexical":"Riyad","offsetInTicks":377300000,"offset":"PT37.73S","durationInTicks":13500000,"duration":"PT1.35S"}]},{"confidence":0.6153370141983032,"lexical":"the riyad","itn":"the riyad","maskedItn":"the riyad","display":"the Riyad","words":[{"confidence":0.38714462518692017,"lexical":"the","offsetInTicks":377500000,"offset":"PT37.75S","durationInTicks":800000,"duration":"PT0.08S"},{"confidence":0.6317668557167053,"lexical":"Riyad","offsetInTicks":378400000,"offset":"PT37.84S","durationInTicks":12400000,"duration":"PT1.24S"}]},{"confidence":0.1274680346250534,"lexical":"riyad riyad","itn":"riyad riyad","maskedItn":"riyad riyad","display":"Riyad Riyad","words":[{"confidence":0.07970714569091797,"lexical":"Riyad","offsetInTicks":377300000,"offset":"PT37.73S","durationInTicks":7000000,"duration":"PT0.7S"},{"confidence":0.17963764071464539,"lexical":"Riyad","offsetInTicks":384400000,"offset":"PT38.44S","durationInTicks":6400000,"duration":"PT0.64S"}]},{"confidence":0.5736846327781677,"lexical":"i riyad","itn":"i riyad","maskedItn":"i riyad","display":"i Riyad","words":[{"confidence":0.4975779354572296,"lexical":"i","offsetInTicks":377900000,"offset":"PT37.79S","durationInTicks":200000,"duration":"PT0.02S"},{"confidence":0.5754824876785278,"lexical":"Riyad","offsetInTicks":378200000,"offset":"PT37.82S","durationInTicks":12600000,"duration":"PT1.26S"}]},{"confidence":0.604779064655304,"lexical":"we riyad","itn":"we riyad","maskedItn":"we riyad","display":"we Riyad","words":[{"confidence":0.3357701599597931,"lexical":"we","offsetInTicks":377500000,"offset":"PT37.75S","durationInTicks":600000,"duration":"PT0.06S"},{"confidence":0.6196063160896301,"lexical":"Riyad","offsetInTicks":378200000,"offset":"PT37.82S","durationInTicks":12600000,"duration":"PT1.26S"}]}]}]} \ No newline at end of file diff --git a/CustomSpeechLog/Names.wav b/CustomSpeechLog/Names.wav new file mode 100644 index 0000000..77bcdba Binary files /dev/null and b/CustomSpeechLog/Names.wav differ diff --git a/README.org b/README.org index 67284c7..b8bbd40 100644 --- a/README.org +++ b/README.org @@ -1,12 +1,42 @@ -Welcome to LT2216/LT2816 V24 Dialogue systems 1 official repository. +Fork from The course LT2216/LT2816 V24 Dialogue systems 1 official repository. -Theory: -- [[Lectures/js.org][Introduction to JavaScript]] - -Practice: -- [[Exercises/javascript-exercises.md][Exercise 1]] - Lab I (see Canvas) - [[Labs/lab2.org][Lab II]] - [[Labs/lab3.org][Lab III]] - [[Labs/lab4.org][Lab IV]] - Lab V + +Implement the dialogue management using Azure. +The coding files I implemented are in /Code/dm*.js + +Regarding Lab5 I implemented a Five2one Meditation Game. +** Technicalities +- Development IDEA: Visual Studio Code +- Voice resources: Azure speech studio +- Implement the transcation between every work-state using state charts. +- HTML for the UI and Javascript for work-flow logic. +- Combine the ASR and TTS with Speechstate. +- Xstate as an inspector for making state transactions visual. + +** What does it do? +- The 54321 method is a simple grounding technique often used to alleviate anxiety or stress by bringing attention back to the present moment. +- There are five missions for this game, every mission with one star. +- Users tell their stress level from 0 to 10 before and after playing this game. +- Finally, finishing five missions will get five star. +- The stress level changes will be calculated. + + +** Challenges +How to update HTML element when the speechstate changes. + +Use ‘cond’ or ‘guard’ for condition judge. +- Solution_a: +entry: [{type: "say", params: `Let’s …. `}, displayCheckMessage, displayStar], +Or actions: [{type: "say", params: `Let’s …. `}, displayCheckMessage] +function displayCheckMessage() { +document.getElementById('textContent').textContent = "Let's check whether …."} +- Solution_b: +Use ‘guard’ instead of ‘cond’ + + + diff --git a/report-lab2.pdf b/report-lab2.pdf new file mode 100644 index 0000000..86dab27 Binary files /dev/null and b/report-lab2.pdf differ diff --git a/report-lab3.pdf b/report-lab3.pdf new file mode 100644 index 0000000..9108eb4 Binary files /dev/null and b/report-lab3.pdf differ