Merge branch 'lc3' into dev

SmartGlassesManager/SGM_android/SmartGlassesManager/src/main/cpp/liblc3/liblc3.cpp

@@ -72,6 +72,57 @@ Java_com_augmentos_smartglassesmanager_cpp_L3cCpp_decodeLC3(JNIEnv *env, jclass
 }
 
 
+extern "C" JNIEXPORT jbyteArray JNICALL
+Java_com_augmentos_smartglassesmanager_cpp_L3cCpp_encodeLC3(JNIEnv *env, jclass instance, jbyteArray pcmData) {
+    // Get PCM data from Java
+    jbyte *pcmBytes = env->GetByteArrayElements(pcmData, nullptr);
+    int pcmLength = env->GetArrayLength(pcmData);
+
+    // Frame duration: 10ms
+    int dtUs = 10000;
+    // Sampling rate: 16kHz
+    int srHz = 16000;
+    // Samples per frame
+    uint16_t samplesPerFrame = lc3_frame_samples(dtUs, srHz);
+    // Bytes per frame (16-bit PCM)
+    uint16_t bytesPerFrame = samplesPerFrame * 2;
+
+    // Output encoded frame size (adjustable, but typically 20 bytes for low bitrate)
+    uint16_t encodedFrameSize = 20;
+
+    // Total encoded output size
+    int outputSize = (pcmLength / bytesPerFrame) * encodedFrameSize;
+
+    // Allocate memory for the encoded output
+    unsigned char *encodedData = (unsigned char *)malloc(outputSize);
+
+    // Allocate encoder memory
+    unsigned encoderSize = lc3_encoder_size(dtUs, srHz);
+    void *encMem = malloc(encoderSize);
+    lc3_encoder_t encoder = lc3_setup_encoder(dtUs, srHz, srHz, encMem);
+
+    jsize offset = 0;
+
+    // Loop through the PCM data in frames
+    for (int i = 0; i <= pcmLength - bytesPerFrame; i += bytesPerFrame) {
+        unsigned char *framePcm = reinterpret_cast<unsigned char *>(pcmBytes + i);
+        lc3_encode(encoder, LC3_PCM_FORMAT_S16, framePcm, 1, encodedFrameSize, encodedData + offset);
+        offset += encodedFrameSize;
+    }
+
+    // Convert output to a Java byte array
+    jbyteArray resultArray = env->NewByteArray(outputSize);
+    env->SetByteArrayRegion(resultArray, 0, outputSize, (jbyte *)encodedData);
+
+    // Cleanup
+    env->ReleaseByteArrayElements(pcmData, pcmBytes, JNI_ABORT);
+    free(encMem);
+    free(encodedData);
+
+    return resultArray;
+}
+
+
 extern "C" JNIEXPORT jfloatArray JNICALL
 Java_com_example_demo_1ai_1even_cpp_Cpp_rnNoise(JNIEnv *env, jclass clazz,jlong st, jfloatArray input) {
     jfloat *inputArray = env->GetFloatArrayElements(input, NULL);

SmartGlassesManager/SGM_android/SmartGlassesManager/src/main/java/com/augmentos/smartglassesmanager/cpp/L3cCpp.java

@@ -16,6 +16,8 @@ public static void init() {
 
     public static native byte[] decodeLC3(byte[] lc3Data);
 
+    public static native byte[] encodeLC3(byte[] lc3Data);
+
     public static native float[] rnNoise(long st, float[] input);
 
     public static native long createRNNoiseState();

augmentos_cloud/packages/lc3_utils/decodeLC3.ts

@@ -0,0 +1,80 @@
+import * as fs from "fs";
+import * as path from "path";
+
+// Path to the liblc3 WebAssembly file
+const wasmPath: string = path.resolve(__dirname, "../bin/liblc3.wasm");
+
+// Load the WebAssembly module
+async function loadLC3(): Promise<WebAssembly.Exports> {
+    const wasmBuffer = fs.readFileSync(wasmPath);
+    const wasmModule = await WebAssembly.instantiate(wasmBuffer, {});
+    return wasmModule.instance.exports;
+}
+
+// Function to allocate memory in the WASM buffer
+function allocateMemory(lc3: WebAssembly.Exports, size: number): number {
+    const ptr = (lc3.memory as WebAssembly.Memory).grow(Math.ceil(size / (64 * 1024))); // Grow memory in pages
+    return ptr * 64 * 1024; // Convert page index to byte offset
+}
+
+// Decode LC3 audio
+async function decodeLC3(): Promise<void> {
+    const lc3 = await loadLC3();
+
+    // Read LC3 encoded audio file
+    const inputFile: string = path.resolve(__dirname, "audio_chunk_0.raw");
+    const encodedBytes: Buffer = fs.readFileSync(inputFile);
+    const encodedSize: number = encodedBytes.length;
+
+    // LC3 decoding parameters
+    const frameDurationUs = 10000; // 10ms per frame
+    const sampleRateHz = 16000; // 16kHz
+    const outputByteCount = 20; // LC3 compressed bytes per frame
+    const numFrames = Math.floor(encodedSize / outputByteCount);
+    const frameSamples = (lc3.lc3_frame_samples as Function)(frameDurationUs, sampleRateHz);
+    const bytesPerFrame = frameSamples * 2; // 16-bit PCM (2 bytes per sample)
+    const outputSize = numFrames * bytesPerFrame;
+
+    // Allocate memory in WASM
+    const inputPtr = allocateMemory(lc3, encodedSize);
+    const outputPtr = allocateMemory(lc3, outputSize);
+    const decoderSize = (lc3.lc3_decoder_size as Function)(frameDurationUs, sampleRateHz);
+    const decoderPtr = allocateMemory(lc3, decoderSize);
+
+    // Copy input data to WASM memory
+    const memoryBuffer = new Uint8Array((lc3.memory as WebAssembly.Memory).buffer);
+    memoryBuffer.set(encodedBytes, inputPtr);
+
+    // Initialize LC3 decoder
+    (lc3.lc3_setup_decoder as Function)(frameDurationUs, sampleRateHz, sampleRateHz, decoderPtr);
+
+    // Decode frames
+    let outputOffset = 0;
+    for (let i = 0; i < numFrames; i++) {
+        const inputFramePtr = inputPtr + i * outputByteCount;
+        const outputFramePtr = outputPtr + outputOffset;
+
+        (lc3.lc3_decode as Function)(
+            decoderPtr,
+            inputFramePtr,
+            outputByteCount,
+            0, // PCM format: 0 = S16 (signed 16-bit PCM)
+            outputFramePtr,
+            1
+        );
+
+        outputOffset += bytesPerFrame;
+    }
+
+    // Retrieve PCM data
+    const pcmData = Buffer.from(memoryBuffer.slice(outputPtr, outputPtr + outputSize));
+
+    // Write PCM output to file
+    const outputFile = path.resolve(__dirname, "decoded_audio.pcm");
+    fs.writeFileSync(outputFile, pcmData);
+
+    console.log(`Decoded audio written to: ${outputFile}`);
+}
+
+decodeLC3().catch(console.error);
+

augmentos_core/app/src/main/java/com/augmentos/augmentos_core/AsrPlanner.java

@@ -69,6 +69,7 @@ public void onTranscript(SpeechRecOutputEvent event) {
             transcriptsBuffer.add(text);
         }
     }
+
     @Subscribe
     public void onTranslate(TranslateOutputEvent event){
         String fromLanguageCode = event.fromLanguageCode;

augmentos_core/app/src/main/java/com/augmentos/augmentos_core/LocationSystem.java

@@ -30,98 +30,85 @@ public class LocationSystem {
 
     private final Handler locationSendingLoopHandler = new Handler(Looper.getMainLooper());
     private Runnable locationSendingRunnableCode;
-    private final long locationSendTime = 1000 * 10; // define in milliseconds
-
+    private final long locationSendTime = 1000 * 60 * 8; // 8 minutes
 
     public LocationSystem(Context context) {
         this.context = context;
         fusedLocationProviderClient = LocationServices.getFusedLocationProviderClient(context);
-        getUserLocation();
+        setupLocationCallback();
+        scheduleLocationUpdates();
     }
 
-    public void getUserLocation() {
-        if (ActivityCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) != PackageManager.PERMISSION_GRANTED && ActivityCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) != PackageManager.PERMISSION_GRANTED) {
-            // TODO: Consider calling ActivityCompat#requestPermissions here.
-            return;
-        }
-
-        fusedLocationProviderClient.getLastLocation().addOnSuccessListener(new OnSuccessListener<Location>() {
-            @Override
-            public void onSuccess(Location location) {
-                // Got last known location. In some rare situations, this can be null.
-                if (location != null) {
-                    lat = location.getLatitude();
-                    lng = location.getLongitude();
-                }
-            }
-        });
-
-        LocationRequest locationRequest = LocationRequest.create();
-        locationRequest.setPriority(LocationRequest.PRIORITY_HIGH_ACCURACY);
-        locationRequest.setInterval(10000); // 10 seconds
-        locationRequest.setFastestInterval(5000); // 5 seconds
-
+    private void setupLocationCallback() {
         locationCallback = new LocationCallback() {
             @Override
             public void onLocationResult(LocationResult locationResult) {
-                if (locationResult == null) {
-                    return;
-                }
-                for (Location location : locationResult.getLocations()) {
-                    lat = location.getLatitude();
-                    lng = location.getLongitude();
-                }
+                if (locationResult == null || locationResult.getLocations().isEmpty()) return;
+
+                // Get the most recent location
+                Location location = locationResult.getLastLocation();
+                lat = location.getLatitude();
+                lng = location.getLongitude();
+
+                sendLocationToServer();
+                stopLocationUpdates(); // Turn off GPS after successful fix
             }
         };
+    }
 
-        if (ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == PackageManager.PERMISSION_GRANTED || ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == PackageManager.PERMISSION_GRANTED) {
-            fusedLocationProviderClient.requestLocationUpdates(locationRequest, locationCallback, null /* Looper */);
-        }
+    public void startLocationSending() {
+        scheduleLocationUpdates();
     }
 
-    public void stopLocationUpdates() {
-        if (fusedLocationProviderClient != null && locationCallback != null) {
-            fusedLocationProviderClient.removeLocationUpdates(locationCallback);
+    public void requestLocationUpdate() {
+        if (ActivityCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) != PackageManager.PERMISSION_GRANTED) {
+            return;
         }
 
-        locationSendingLoopHandler.removeCallbacks(locationSendingRunnableCode);
-        locationSendingLoopHandler.removeCallbacksAndMessages(null);
-    }
-
-    public double getNewLat() {
-        if (latestAccessedLat == lat) return -1;
+        LocationRequest locationRequest = LocationRequest.create();
+        locationRequest.setPriority(LocationRequest.PRIORITY_HIGH_ACCURACY);
+        locationRequest.setInterval(10000);  // Keep GPS on for a reasonable time to get a fix
+        locationRequest.setFastestInterval(5000);
+        locationRequest.setMaxWaitTime(60000); // Wait up to 60 seconds if needed for a fix
 
-        latestAccessedLat = lat;
-        return latestAccessedLat;
+        fusedLocationProviderClient.requestLocationUpdates(locationRequest, locationCallback, Looper.getMainLooper());
     }
 
-    public double getNewLng() {
-        if (latestAccessedLong == lng) return -1;
-
-        latestAccessedLong = lng;
-        return latestAccessedLong;
+    public void stopLocationUpdates() {
+        if (fusedLocationProviderClient != null && locationCallback != null) {
+            fusedLocationProviderClient.removeLocationUpdates(locationCallback);
+        }
     }
 
-    public void startLocationSending() {
-        locationSendingLoopHandler.removeCallbacksAndMessages(this);
-
+    private void scheduleLocationUpdates() {
         locationSendingRunnableCode = new Runnable() {
             @Override
             public void run() {
-                    sendLocationToServer();
-                locationSendingLoopHandler.postDelayed(this, locationSendTime);
+                requestLocationUpdate(); // Turn on GPS and request a fix
+                locationSendingLoopHandler.postDelayed(this, locationSendTime); // Schedule next run
             }
         };
         locationSendingLoopHandler.post(locationSendingRunnableCode);
     }
 
-    private void sendLocationToServer(){
+    private void sendLocationToServer() {
         double latitude = getNewLat();
         double longitude = getNewLng();
 
-        if(latitude == -1 && longitude == -1) return;
+        if (latitude == -1 && longitude == -1) return;
 
         ServerComms.getInstance().sendLocationUpdate(latitude, longitude);
     }
-}
 
+    public double getNewLat() {
+        if (latestAccessedLat == lat) return -1;
+        latestAccessedLat = lat;
+        return latestAccessedLat;
+    }
+
+    public double getNewLng() {
+        if (latestAccessedLong == lng) return -1;
+        latestAccessedLong = lng;
+        return latestAccessedLong;
+    }
+}

augmentos_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/eventbusmessages/LC3AudioChunkNewEvent.java

@@ -0,0 +1,9 @@
+package com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages;
+
+public class LC3AudioChunkNewEvent {
+    public byte [] thisChunk;
+
+    public LC3AudioChunkNewEvent(byte [] thisChunk){
+        this.thisChunk = thisChunk;
+    }
+}

augmentos_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassescommunicators/EvenRealitiesG1SGC.java

@@ -38,6 +38,7 @@
 import java.util.zip.CRC32;
 import java.nio.ByteBuffer;
 
+import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.LC3AudioChunkNewEvent;
 import com.augmentos.augmentos_core.smarterglassesmanager.smartglassesconnection.SmartGlassesAndroidService;
 import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.HeadUpAngleEvent;
 import com.augmentos.augmentos_core.smarterglassesmanager.utils.G1FontLoader;
@@ -401,6 +402,7 @@ public void onCharacteristicChanged(BluetoothGatt gatt, BluetoothGattCharacteris
                                 // Log.d(TAG, "Audio data received. Seq: " + seq + ", from: " + deviceName + ", length: " + pcmData.length);
                                 if (shouldRunOnboardMic) {
                                     EventBus.getDefault().post(new AudioChunkNewEvent(pcmData));
+                                    EventBus.getDefault().post(new LC3AudioChunkNewEvent(lc3));
                                 }
                             } else {
 //                                Log.d(TAG, "Lc3 Audio data received. Seq: " + seq + ", Data: " + Arrays.toString(lc3) + ", from: " + deviceName);
@@ -518,6 +520,9 @@ private void initG1s(BluetoothGatt gatt, String side){
                 Log.d(TAG, "Sending turn off wear detection command");
                 sendDataSequentially(new byte[]{(byte) 0x27, (byte) 0x00});
 
+                Log.d(TAG, "Sending turn off silent mode Command");
+                sendDataSequentially(new byte[]{(byte) 0x03, (byte) 0x0A});
+
                 //debug command
 //                            Log.d(TAG, "Sending debug 0xF4 Command");
 //                            sendDataSequentially(new byte[]{(byte) 0xF4, (byte) 0x01});
@@ -529,7 +534,6 @@ private void initG1s(BluetoothGatt gatt, String side){
                 //setup brightness
                 sendBrightnessCommandHandler.postDelayed(() -> sendBrightnessCommand(brightnessValue, shouldUseAutoBrightness), 10);
 
-
                 // Maybe start MIC streaming
                 setMicEnabled(shouldRunOnboardMic, 10); // Enable the MIC
 

augmentos_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/smartglassesconnection/SmartGlassesRepresentative.java

@@ -14,6 +14,7 @@
 
 import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.AudioChunkNewEvent;
 import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.DisableBleScoAudioEvent;
+import com.augmentos.augmentos_core.smarterglassesmanager.eventbusmessages.LC3AudioChunkNewEvent;
 import com.augmentos.augmentoslib.events.DisplayCustomContentRequestEvent;
 import com.augmentos.augmentoslib.events.DoubleTextWallViewRequestEvent;
 import com.augmentos.augmentoslib.events.HomeScreenEvent;
@@ -42,6 +43,7 @@
 import com.augmentos.augmentos_core.smarterglassesmanager.supportedglasses.SmartGlassesDevice;
 import com.augmentos.augmentoslib.events.TextLineViewRequestEvent;
 import com.augmentos.augmentos_core.smarterglassesmanager.utils.SmartGlassesConnectionState;
+import com.augmentos.smartglassesmanager.cpp.L3cCpp;
 
 //rxjava
 import java.nio.ByteBuffer;
@@ -205,8 +207,14 @@ public void onSuccess(ByteBuffer chunk){
 
     private void receiveChunk(ByteBuffer chunk){
         byte[] audio_bytes = chunk.array();
+
+        //encode as LC3
+        byte[] lc3Data = L3cCpp.encodeLC3(chunk.array());
+        Log.d(TAG, "LC3 Data encoded: " + lc3Data.toString());
+
         //throw off new audio chunk event
         EventBus.getDefault().post(new AudioChunkNewEvent(audio_bytes));
+        EventBus.getDefault().post(new LC3AudioChunkNewEvent(lc3Data));
     }
 
     public void destroy(){

augmentos_core/app/src/main/java/com/augmentos/augmentos_core/smarterglassesmanager/speechrecognition/SpeechRecFramework.java

@@ -13,6 +13,8 @@ public abstract class SpeechRecFramework {
     public abstract void destroy();
     public abstract void ingestAudioChunk(byte [] audioChunk);
 
+    public abstract void ingestLC3AudioChunk(byte [] audioChunk);
+
     public void pauseAsr(boolean pauseAsrFlag){
         this.pauseAsrFlag = pauseAsrFlag;
     }