Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

v1.1 ios #23

Merged
merged 4 commits into from
Oct 2, 2024
Merged

v1.1 ios #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d7cbfe6
ios demo
laves Sep 13, 2024
98fa7cd
v1.1 picollm changes
ksyeo1010 Oct 2, 2024
8b42514
remove return
ksyeo1010 Oct 2, 2024
35d380d
fix lint
ksyeo1010 Oct 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 23 additions & 11 deletions recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ class AudioPlayerStream {

private var pcmBuffers = [AVAudioPCMBuffer]()
public var isPlaying = false
public var isStopped = false

init(sampleRate: Double) throws {
let audioSession = AVAudioSession.sharedInstance()
try audioSession.setCategory(.playback, mode: .default)
try audioSession.setCategory(.playAndRecord, options: [.mixWithOthers, .allowBluetooth])
try audioSession.setActive(true)

let format = AVAudioFormat(
Expand All @@ -38,7 +39,10 @@ class AudioPlayerStream {
try engine.start()
}

func playStreamPCM(_ pcmData: [Int16], completion: @escaping (Bool) -> Void) {
func playStreamPCM(_ pcmData: [Int16]) throws {
if isStopped {
return
}
let audioBuffer = AVAudioPCMBuffer(
pcmFormat: playerNode.outputFormat(forBus: 0), frameCapacity: AVAudioFrameCount(pcmData.count))!

Expand All @@ -56,32 +60,40 @@ class AudioPlayerStream {
}

pcmBuffers.append(audioBuffer)

if !engine.isRunning {
try engine.start()
}
if !isPlaying {
playNextPCMBuffer(completion: completion)
} else {
completion(true)
playNextPCMBuffer()
}
}

private func playNextPCMBuffer(completion: @escaping (Bool) -> Void) {
private func playNextPCMBuffer() {
if isStopped {
return
}
guard let pcmData = pcmBuffers.first else {
isPlaying = false
completion(false)
return
}
pcmBuffers.removeFirst()

playerNode.scheduleBuffer(pcmData) { [weak self] in
self?.playNextPCMBuffer(completion: completion)
self?.playNextPCMBuffer()
}

playerNode.play()
isPlaying = true
completion(true)
}

func resetAudioPlayer() {
isStopped = false
isPlaying = false
}

func stopStreamPCM() {
playerNode.stop()
engine.stop()
isStopped = true
pcmBuffers.removeAll()
}
}
72 changes: 44 additions & 28 deletions recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ You can download directly to your device or airdrop from a Mac.

private func streamCallback(completion: String) {
DispatchQueue.main.async { [self] in
if self.stopPhrases.contains(completion) {
if self.stopPhrases.contains(completion) || chatState != .GENERATE {
return
}

Expand Down Expand Up @@ -222,23 +222,33 @@ You can download directly to your device or airdrop from a Mac.
streamCallback: streamCallback)

try dialog!.addLLMResponse(content: result.completion)

DispatchQueue.main.async { [self] in
if result.endpoint == .interrupted {
statusText = "Listening..."
chatText.append(Message(speaker: "You:", msg: ""))
chatState = .STT

promptText = ""
enableGenerateButton = true
} else {
statusText = ViewModel.statusTextDefault
chatState = .WAKEWORD

promptText = ""
enableGenerateButton = true
}
}
} catch {
DispatchQueue.main.async { [self] in
errorMessage = "\(error.localizedDescription)"
}
}

DispatchQueue.main.async { [self] in
statusText = ViewModel.statusTextDefault
chatState = .WAKEWORD

promptText = ""
enableGenerateButton = true
}
}

DispatchQueue.global(qos: .userInitiated).async { [self] in
do {
audioStream!.resetAudioPlayer()
let orcaStream = try self.orca!.streamOpen()

var warmup = true
Expand All @@ -262,32 +272,24 @@ You can download directly to your device or airdrop from a Mac.
if warmup {
warmupBuffer.append(contentsOf: pcm!)
if warmupBuffer.count >= (1 * orca!.sampleRate!) {
audioStream!.playStreamPCM(warmupBuffer, completion: { isPlaying in
if !isPlaying {
self.startAudioRecording()
}
})
try audioStream!.playStreamPCM(pcm!)
warmupBuffer.removeAll()
warmup = false
}
} else {
audioStream!.playStreamPCM(pcm!, completion: {_ in })
try audioStream!.playStreamPCM(pcm!)
}
}
}
}

if !warmupBuffer.isEmpty {
audioStream!.playStreamPCM(warmupBuffer, completion: { isPlaying in
if !isPlaying {
self.startAudioRecording()
}
})
try audioStream!.playStreamPCM(warmupBuffer)
}

let pcm = try orcaStream.flush()
if pcm != nil {
audioStream!.playStreamPCM(pcm!, completion: {_ in})
try audioStream!.playStreamPCM(pcm!)
}
orcaStream.close()
} catch {
Expand All @@ -298,25 +300,40 @@ You can download directly to your device or airdrop from a Mac.
}
}

public func interrupt() {
do {
audioStream!.stopStreamPCM()
try picollm?.interrupt()
} catch {
DispatchQueue.main.async { [self] in
errorMessage = "\(error.localizedDescription)"
}
}
}

public func clearText() {
promptText = ""
chatText.removeAll()
}

private func audioCallback(frame: [Int16]) {
do {
if audioStream?.isPlaying ?? false {
return
}
if chatState == .WAKEWORD {
let keyword = try self.porcupine!.process(pcm: frame)
if keyword != -1 {
let keywordIndex = try self.porcupine!.process(pcm: frame)
if keywordIndex == 0 {
DispatchQueue.main.async { [self] in
statusText = "Listening..."
chatText.append(Message(speaker: "You:", msg: ""))
chatState = .STT
}
}
} else if chatState == .GENERATE {
let keywordIndex = try self.porcupine!.process(pcm: frame)
if keywordIndex == 0 {
DispatchQueue.main.async { [self] in
self.interrupt()
}
}
} else if chatState == .STT {
var (transcription, endpoint) = try self.cheetah!.process(frame)
if endpoint {
Expand All @@ -329,9 +346,8 @@ You can download directly to your device or airdrop from a Mac.
}
if endpoint {
DispatchQueue.main.async { [self] in
statusText = "Generating..."
statusText = "Generating, Say `Picovoice` to interrupt"
chatState = .GENERATE
stopAudioRecording()
self.generate()
}
}
Expand Down
2 changes: 1 addition & 1 deletion recipes/llm-voice-assistant/ios/Podfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ platform :ios, '16.0'
target 'LLMVoiceAssistantDemo' do
pod 'Porcupine-iOS', '~> 3.0.1'
pod 'Cheetah-iOS', '~> 2.0.0'
pod 'picoLLM-iOS', '~> 1.0.0'
pod 'picoLLM-iOS', '~> 1.1.0'
pod 'Orca-iOS', '~> 1.0.0'
pod 'ios-voice-processor', '~> 1.1.0'
end
8 changes: 4 additions & 4 deletions recipes/llm-voice-assistant/ios/Podfile.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ PODS:
- Cheetah-iOS (2.0.0)
- ios-voice-processor (1.1.0)
- Orca-iOS (1.0.0)
- picoLLM-iOS (1.0.0)
- picoLLM-iOS (1.1.0)
- Porcupine-iOS (3.0.1):
- ios-voice-processor (~> 1.1.0)

DEPENDENCIES:
- Cheetah-iOS (~> 2.0.0)
- ios-voice-processor (~> 1.1.0)
- Orca-iOS (~> 1.0.0)
- picoLLM-iOS (~> 1.0.0)
- picoLLM-iOS (~> 1.1.0)
- Porcupine-iOS (~> 3.0.1)

SPEC REPOS:
Expand All @@ -25,9 +25,9 @@ SPEC CHECKSUMS:
Cheetah-iOS: d98a5edcbf3b74dda6027aeac6a8c0f5997a47a2
ios-voice-processor: 8e32d7f980a06d392d128ef1cd19cf6ddcaca3c1
Orca-iOS: d50a0dbbf596f20c6c2e2f727f20f72ac012aa0e
picoLLM-iOS: 02cdb501b4beb74a9c1dea29d5cf461d65ea4a6c
picoLLM-iOS: dc03cd7e992c702ff34c667f9a35dd9a8084c061
Porcupine-iOS: 6d69509fa587f3ac0be1adfefb48e0c6ce029fff

PODFILE CHECKSUM: 64580b5dbb7bc16cb10af3dc7da63609228fe397
PODFILE CHECKSUM: 1cc2ff3bc3e1abc97fbf7a3b3910e8d2b805f8b7

COCOAPODS: 1.15.2
Loading