Skip to content

Commit

Permalink
Camera Capture.
Browse files Browse the repository at this point in the history
  • Loading branch information
chigkim committed May 20, 2024
1 parent 260c62b commit c31cf53
Show file tree
Hide file tree
Showing 13 changed files with 245 additions and 53 deletions.
1 change: 1 addition & 0 deletions Shortcuts.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
{"keyName":"⌃⇧⌘S","key":1,"modifiers":4864,"name":"Settings"},
{"keyName":"⌃⇧⌘W","key":13,"modifiers":4864,"name":"OCR Window"},
{"keyName":"⌃⇧⌘V","key":9,"modifiers":4864,"name":"OCR VOCursor"},
{"keyName":"⌃⇧⌘C","key":8,"modifiers":4864,"name":"Capture Camera"},
{"keyName":"⌃⇧⌘E","key":14,"modifiers":4864,"name":"Explore"},
{"keyName":"⌃⇧⌘A","key":0,"modifiers":4864,"name":"Ask"},
{"keyName":"⌃⇧⌘R","key":15,"modifiers":4864,"name":"Realtime OCR"},
Expand Down
12 changes: 8 additions & 4 deletions VOCR.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
EA9EB89E28EA5B0A00FE6210 /* Utils.swift in Sources */ = {isa = PBXBuildFile; fileRef = EA9EB89D28EA5B0A00FE6210 /* Utils.swift */; };
EAAD42AE298596FB00BE5D98 /* com.chikim.VOCR.plist in Resources */ = {isa = PBXBuildFile; fileRef = EAAD42AD298596FB00BE5D98 /* com.chikim.VOCR.plist */; };
EAC1C94B2B59D96200FBE97D /* AutoUpdateManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = EAC1C94A2B59D96200FBE97D /* AutoUpdateManager.swift */; };
EAC9ECD92BFB8D6200D61F00 /* MacCamera.swift in Sources */ = {isa = PBXBuildFile; fileRef = EAC9ECD82BFB8D6200D61F00 /* MacCamera.swift */; };
EAD0E5A92B57942F004E69AD /* FileLogger.swift in Sources */ = {isa = PBXBuildFile; fileRef = EAD0E5A82B57942F004E69AD /* FileLogger.swift */; };
EAD9AAE62731075600473054 /* HotKey in Frameworks */ = {isa = PBXBuildFile; productRef = EAD9AAE52731075600473054 /* HotKey */; };
EAD9AAE927310A3900473054 /* SoundpipeAudioKit in Frameworks */ = {isa = PBXBuildFile; productRef = EAD9AAE827310A3900473054 /* SoundpipeAudioKit */; };
Expand Down Expand Up @@ -67,6 +68,7 @@
EA9EB89D28EA5B0A00FE6210 /* Utils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Utils.swift; sourceTree = "<group>"; };
EAAD42AD298596FB00BE5D98 /* com.chikim.VOCR.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = com.chikim.VOCR.plist; sourceTree = "<group>"; };
EAC1C94A2B59D96200FBE97D /* AutoUpdateManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AutoUpdateManager.swift; sourceTree = "<group>"; };
EAC9ECD82BFB8D6200D61F00 /* MacCamera.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MacCamera.swift; sourceTree = "<group>"; };
EAD0E5A82B57942F004E69AD /* FileLogger.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileLogger.swift; sourceTree = "<group>"; };
EAD9CFD12355E11600E7F594 /* say.scpt */ = {isa = PBXFileReference; lastKnownFileType = file; path = say.scpt; sourceTree = "<group>"; };
EADE59B1232EB09A00E2F65A /* VOCR.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VOCR.app; sourceTree = BUILT_PRODUCTS_DIR; };
Expand Down Expand Up @@ -134,6 +136,7 @@
EADE59B3232EB09A00E2F65A /* VOCR */ = {
isa = PBXGroup;
children = (
EAC9ECD82BFB8D6200D61F00 /* MacCamera.swift */,
EADE59B6232EB09A00E2F65A /* AboutViewController.swift */,
EA2DA1AB232EB1D600D31031 /* Accessibility.swift */,
EADE59B4232EB09A00E2F65A /* AppDelegate.swift */,
Expand Down Expand Up @@ -272,6 +275,7 @@
EA5581EF2B5AF6F800408F92 /* KeychainManager.swift in Sources */,
EA9AF47D2B54BD3A001B1BBE /* Ollama.swift in Sources */,
EAF9C3C82354A66200D4D77C /* Settings.swift in Sources */,
EAC9ECD92BFB8D6200D61F00 /* MacCamera.swift in Sources */,
EA1F642C2352503D0089C967 /* Player.swift in Sources */,
EA0C88682B4F8F4B00F2E920 /* ShortcutsWindowController.swift in Sources */,
EA9AF47F2B54D03D001B1BBE /* Engines.swift in Sources */,
Expand Down Expand Up @@ -419,7 +423,7 @@
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 33;
CURRENT_PROJECT_VERSION = 34;
DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_TEAM = 9N598S2535;
ENABLE_HARDENED_RUNTIME = YES;
Expand All @@ -431,7 +435,7 @@
"$(inherited)",
"@executable_path/../Frameworks",
);
MARKETING_VERSION = "2.0.0-beta.2";
MARKETING_VERSION = "2.0.0-beta.3";
PRODUCT_BUNDLE_IDENTIFIER = com.chikim.VOCR;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_VERSION = 5.0;
Expand All @@ -446,7 +450,7 @@
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 33;
CURRENT_PROJECT_VERSION = 34;
DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_TEAM = 9N598S2535;
ENABLE_HARDENED_RUNTIME = YES;
Expand All @@ -458,7 +462,7 @@
"$(inherited)",
"@executable_path/../Frameworks",
);
MARKETING_VERSION = "2.0.0-beta.2";
MARKETING_VERSION = "2.0.0-beta.3";
PRODUCT_BUNDLE_IDENTIFIER = com.chikim.VOCR;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_VERSION = 5.0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/sparkle-project/Sparkle",
"state" : {
"revision" : "47d3d90aee3c52b6f61d04ceae426e607df62347",
"version" : "2.5.2"
"revision" : "41847a58cdef7506b257591fcca6f9495df591d4",
"version" : "2.6.2"
}
}
],
Expand Down
12 changes: 8 additions & 4 deletions VOCR/AutoUpdateManager.swift
Original file line number Diff line number Diff line change
Expand Up @@ -27,20 +27,24 @@ class AutoUpdateManager: NSObject, SPUUpdaterDelegate, SPUStandardUserDriverDele
if let automaticallyChecksForUpdates = self.updaterController?.updater.automaticallyChecksForUpdates, automaticallyChecksForUpdates {
self.updaterController?.updater.checkForUpdatesInBackground()
}
}
}

func checkForUpdates() {
updaterController?.checkForUpdates(nil)
}

func allowedChannels(for updater: SPUUpdater) -> Set<String> {
// if Settings.preRelease {
// updater.updateCheckInterval = 3600
// } else {
// updater.updateCheckInterval = 3600*24
// }

if Settings.preRelease {
log("Download pre-release.")
updater.updateCheckInterval = 3600
log("Download pre-release.")
return Set(["pre"])
} else {
log("No pre-release.")
updater.updateCheckInterval = 3600*24
return Set()
}
}
Expand Down
2 changes: 2 additions & 0 deletions VOCR/Info.plist
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>NSCameraUsageDescription</key>
<string>Capture a photo to VOCR to use.</string>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleDocumentTypes</key>
Expand Down
120 changes: 120 additions & 0 deletions VOCR/MacCamera.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
//
// Camera.swift
// VOCR
//
// Created by Chi Kim on 10/12/19.
// Copyright © 2019 Chi Kim. All rights reserved.
//

import AVFoundation
import Cocoa
import UniformTypeIdentifiers
class MacCamera:NSObject, AVCapturePhotoCaptureDelegate {

static let shared = MacCamera()
var captureSession:AVCaptureSession!
var cameraOutput:AVCapturePhotoOutput!
var deviceName = "Unknown Camera"

func isCameraAllowed() -> Bool {
let cameraPermissionStatus = AVCaptureDevice.authorizationStatus(for: .video)
switch cameraPermissionStatus {
case .authorized:
print("Already Authorized")
return true
case .denied:
print("denied")
return false
case .restricted:
print("restricted")
return false
default:
print("Ask permission")

var access = false
AVCaptureDevice.requestAccess(for: .video) { granted in
if granted == true {
print("User granted")
} else {
print("User denied")
}
access = granted
}
return access
}
}

func getCamera() -> AVCaptureDevice? {
let devices = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera,.externalUnknown], mediaType: .video, position: .unspecified).devices
for device in devices {
if Settings.camera == device.localizedName {
return device
}
}
if let device = AVCaptureDevice.default(for: .video) {
return device
}
return nil
}

func takePicture() {
guard let device = getCamera() else {
Accessibility.speakWithSynthesizer("No camera available.")
return
}

for c in stride(from: 3, to: 1, by: -1) {
Accessibility.speak("\(c)")
sleep(1)
}
Accessibility.speak("1")
captureSession = AVCaptureSession()
captureSession.sessionPreset = AVCaptureSession.Preset.photo
cameraOutput = AVCapturePhotoOutput()
if let input = try? AVCaptureDeviceInput(device: device) {
if (captureSession.canAddInput(input)) {
captureSession.addInput(input)
if (captureSession.canAddOutput(cameraOutput)) {
captureSession.addOutput(cameraOutput)
captureSession.startRunning()
let settings = AVCapturePhotoSettings()
deviceName = device.localizedName
Accessibility.speak(deviceName)
cameraOutput.capturePhoto(with: settings, delegate: self)
}
} else {
print("issue here : captureSesssion.canAddInput")
}
} else {
print("some problem here")
}
}

func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
captureSession.stopRunning()
print("photo captured")
if let error = error {
debugPrint(error)
}

if let dataImage = photo.fileDataRepresentation() {
let dataProvider = CGDataProvider(data: dataImage as CFData)
if let cgImage = CGImage(jpegDataProviderSource: dataProvider!, decode: nil, shouldInterpolate: true, intent: .defaultIntent) {
NSSound(contentsOfFile: "/System/Library/Components/CoreAudio.component/Contents/SharedSupport/SystemSounds/system/Shutter.aif", byReference: true)?.play()
classify(cgImage:cgImage)
Navigation.mode = .CAMERA
Navigation.appName = deviceName
Navigation.cgImage = cgImage
}
} else {
print("some error here")
}
}

@discardableResult func writeCGImage(_ image: CGImage, to destinationURL: URL) -> Bool {
guard let destination = CGImageDestinationCreateWithURL(destinationURL as CFURL, UTType.png.identifier as CFString, 1, nil) else { return false }
CGImageDestinationAddImage(destination, image, nil)
return CGImageDestinationFinalize(destination)
}

}
3 changes: 3 additions & 0 deletions VOCR/Navigation.swift
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ enum Navigation {
enum Mode: Int, CaseIterable {
case WINDOW = 0
case VOCURSOR = 1
case CAMERA = 2
func next() -> Mode {
let allCases = Mode.allCases
let nextIndex = (self.rawValue + 1) % allCases.count
Expand All @@ -27,6 +28,8 @@ enum Navigation {
return "Window"
case .VOCURSOR:
return "VOCursor"
case .CAMERA:
return "Camera"
}
}
}
Expand Down
39 changes: 34 additions & 5 deletions VOCR/Settings.swift
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import Cocoa
import AudioKit
import AVFoundation

enum Settings {

Expand All @@ -29,6 +30,7 @@ enum Settings {
static var engine: Engines = .ollama
static var writeLog = false
static var preRelease = false
static var camera = "Unknown"

static var allSettings: [(title: String, action: Selector, value: Bool)] {
return [
Expand Down Expand Up @@ -92,11 +94,15 @@ enum Settings {
let engineMenuItem = NSMenuItem(title: "Engine", action: nil, keyEquivalent: "")
engineMenuItem.submenu = engineMenu
settingsMenu.addItem(engineMenuItem)

let soundOutputMenuItem = NSMenuItem(title: "Sound Output...", action: #selector(target.chooseOutput(_:)), keyEquivalent: "")
soundOutputMenuItem.target = target
settingsMenu.addItem(soundOutputMenuItem)


let cameraMenuItem = NSMenuItem(title: "Choose Camera...", action: #selector(target.chooseCamera(_:)), keyEquivalent: "")
cameraMenuItem.target = target
settingsMenu.addItem(cameraMenuItem)

let shortcutsMenuItem = NSMenuItem(title: "Shortcuts...", action: #selector(target.openShortcutsWindow(_:)), keyEquivalent: "")
shortcutsMenuItem.target = target
settingsMenu.addItem(shortcutsMenuItem)
Expand All @@ -110,7 +116,7 @@ enum Settings {
menu.addItem(settingsMenuItem)

if Navigation.cgImage != nil {
let saveScreenshotMenuItem = NSMenuItem(title: "Save Screenschot", action: #selector(target.saveScreenShot(_:)), keyEquivalent: "s")
let saveScreenshotMenuItem = NSMenuItem(title: "Save Latest Image", action: #selector(target.saveLastImage(_:)), keyEquivalent: "s")
saveScreenshotMenuItem.target = target
menu.addItem(saveScreenshotMenuItem)
}
Expand Down Expand Up @@ -161,6 +167,7 @@ enum Settings {
menu.addItem(NSMenuItem(title: "Quit", action: #selector(NSApplication.terminate(_:)), keyEquivalent: "q"))
return menu
}

static func installMouseMonitor() {
self.eventMonitor = NSEvent.addGlobalMonitorForEvents(
matching: [NSEvent.EventTypeMask.leftMouseDown],
Expand Down Expand Up @@ -244,6 +251,9 @@ enum Settings {
if let mode = defaults.string(forKey: "mode") {
Settings.mode = mode
}
if let camera = defaults.string(forKey: "camera") {
Settings.camera = camera
}
if let prompt = defaults.string(forKey: "prompt") {
Settings.prompt = prompt
}
Expand All @@ -267,6 +277,7 @@ enum Settings {
defaults.set(Settings.prompt, forKey:"prompt")
defaults.set(Settings.systemPrompt, forKey:"systemPrompt")
defaults.set(Settings.mode, forKey:"mode")
defaults.set(Settings.camera, forKey:"camera")
}


Expand Down Expand Up @@ -374,7 +385,25 @@ class MenuHandler: NSObject {
try! Player.shared.engine.setDevice(AudioEngine.outputDevices[n])
try! Player.shared.engine.start()
}


@objc func chooseCamera(_ sender: Any?) {
let devices = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera,.externalUnknown], mediaType: .video, position: .unspecified).devices
if devices.count>1 {
let alert = NSAlert()
alert.alertStyle = .informational
alert.messageText = "Camera"
alert.informativeText = "Choose a camera for VOCR to use."
for device in devices {
alert.addButton(withTitle: device.localizedName)
}
let modalResult = alert.runModal()
hide()
let n = modalResult.rawValue-1000
Settings.camera = devices[n].localizedName
Settings.save()
}
}

@objc func saveResult(_ sender: NSMenuItem) {
let savePanel = NSSavePanel()
savePanel.allowedContentTypes = [.text]
Expand Down Expand Up @@ -406,7 +435,7 @@ class MenuHandler: NSObject {

}

@objc func saveScreenShot(_ sender: NSMenuItem) {
@objc func saveLastImage(_ sender: NSMenuItem) {
if let cgImage = Navigation.cgImage {
try! saveImage(cgImage)
}
Expand Down
Loading

0 comments on commit c31cf53

Please sign in to comment.