audio: Added a postmix callback to logical devices.

You can see it in action in testaudio by mousing over a logical device; it
will show a visualizer for the current PCM (whatever is currently being
recorded on a capture device, or whatever is being mixed for output on
playback devices).

Fixes libsdl-org#8122.

include/SDL3/SDL_audio.h

@@ -1126,6 +1126,73 @@ extern DECLSPEC void SDLCALL SDL_DestroyAudioStream(SDL_AudioStream *stream);
  */
 extern DECLSPEC SDL_AudioStream *SDLCALL SDL_OpenAudioDeviceStream(SDL_AudioDeviceID devid, const SDL_AudioSpec *spec, SDL_AudioStreamCallback callback, void *userdata);
 
+
+/**
+ * A callback that fires when data is about to be fed to an audio device.
+ *
+ * This is useful for accessing the final mix, perhaps for writing a
+ * visualizer or applying a final effect to the audio data before playback.
+ *
+ * \sa SDL_SetAudioDevicePostmixCallback
+ */
+typedef void (SDLCALL *SDL_AudioPostmixCallback)(void *userdata, const SDL_AudioSpec *spec, float *buffer, int buflen);
+
+/**
+ * Set a callback that fires when data is about to be fed to an audio device.
+ *
+ * This is useful for accessing the final mix, perhaps for writing a
+ * visualizer or applying a final effect to the audio data before playback.
+ *
+ * The buffer is the final mix of all bound audio streams on an opened
+ * device; this callback will fire regularly for any device that is both
+ * opened and unpaused. If there is no new data to mix, either because no
+ * streams are bound to the device or all the streams are empty, this
+ * callback will still fire with the entire buffer set to silence.
+ *
+ * This callback is allowed to make changes to the data; the contents of
+ * the buffer after this call is what is ultimately passed along to the
+ * hardware.
+ *
+ * The callback is always provided the data in float format (values from
+ * -1.0f to 1.0f), but the number of channels or sample rate may be
+ * different than the format the app requested when opening the device; SDL
+ * might have had to manage a conversion behind the scenes, or the playback
+ * might have jumped to new physical hardware when a system default changed,
+ * etc. These details may change between calls. Accordingly, the size of the
+ * buffer might change between calls as well.
+ *
+ * This callback can run at any time, and from any thread; if you need to
+ * serialize access to your app's data, you should provide and use a mutex or
+ * other synchronization device.
+ *
+ * All of this to say: there are specific needs this callback can fulfill,
+ * but it is not the simplest interface. Apps should generally provide audio
+ * in their preferred format through an SDL_AudioStream and let SDL handle
+ * the difference.
+ *
+ * This function is extremely time-sensitive; the callback should do the
+ * least amount of work possible and return as quickly as it can. The longer
+ * the callback runs, the higher the risk of audio dropouts or other problems.
+ *
+ * This function will block until the audio device is in between iterations,
+ * so any existing callback that might be running will finish before this
+ * function sets the new callback and returns.
+ *
+ * Setting a NULL callback function disables any previously-set callback.
+ *
+ * \param devid The ID of an opened audio device.
+ * \param callback A callback function to be called. Can be NULL.
+ * \param userdata App-controlled pointer passed to callback. Can be NULL.
+ * \returns zero on success, -1 on error; call SDL_GetError() for more
+ *          information.
+ *
+ * \threadsafety It is safe to call this function from any thread.
+ *
+ * \since This function is available since SDL 3.0.0.
+ */
+extern DECLSPEC int SDLCALL SDL_SetAudioPostmixCallback(SDL_AudioDeviceID devid, SDL_AudioPostmixCallback callback, void *userdata);
+
+
 /**
  * Load the audio data of a WAVE file into memory.
  *

src/audio/SDL_audio.c

@@ -715,6 +715,7 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device)
         // can we do a basic copy without silencing/mixing the buffer? This is an extremely likely scenario, so we special-case it.
         const SDL_bool simple_copy = device->logical_devices &&   // there's a logical device
                                      !device->logical_devices->next &&  // there's only _ONE_ logical device
+                                     !device->logical_devices->postmix &&  // there isn't a postmix callback
                                      !SDL_AtomicGet(&device->logical_devices->paused) &&   // it isn't paused
                                      device->logical_devices->bound_streams &&  // there's a bound stream
                                      !device->logical_devices->bound_streams->next_binding;  // there's only _ONE_ bound stream.
@@ -731,7 +732,7 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device)
                 SDL_memset(device_buffer + br, device->silence_value, buffer_size - br);  // silence whatever we didn't write to.
             }
         } else {  // need to actually mix (or silence the buffer)
-            float *mix_buffer = (float *) ((device->spec.format == SDL_AUDIO_F32) ? device_buffer : device->mix_buffer);
+            float *final_mix_buffer = (float *) ((device->spec.format == SDL_AUDIO_F32) ? device_buffer : device->mix_buffer);
             const int needed_samples = buffer_size / SDL_AUDIO_BYTESIZE(device->spec.format);
             const int work_buffer_size = needed_samples * sizeof (float);
             SDL_AudioSpec outspec;
@@ -742,13 +743,20 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device)
             outspec.channels = device->spec.channels;
             outspec.freq = device->spec.freq;
 
-            SDL_memset(mix_buffer, '\0', buffer_size);  // start with silence.
+            SDL_memset(final_mix_buffer, '\0', work_buffer_size);  // start with silence.
 
             for (SDL_LogicalAudioDevice *logdev = device->logical_devices; logdev != NULL; logdev = logdev->next) {
                 if (SDL_AtomicGet(&logdev->paused)) {
                     continue;  // paused? Skip this logical device.
                 }
 
+                const SDL_AudioPostmixCallback postmix = logdev->postmix;
+                float *mix_buffer = final_mix_buffer;
+                if (postmix) {
+                    mix_buffer = device->postmix_buffer;
+                    SDL_memset(mix_buffer, '\0', work_buffer_size);  // start with silence.
+                }
+
                 for (SDL_AudioStream *stream = logdev->bound_streams; stream != NULL; stream = stream->next_binding) {
                     SDL_SetAudioStreamFormat(stream, NULL, &outspec);
 
@@ -764,12 +772,18 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device)
                         MixFloat32Audio(mix_buffer, (float *) device->work_buffer, br);
                     }
                 }
+
+                if (postmix) {
+                    SDL_assert(mix_buffer == device->postmix_buffer);
+                    postmix(logdev->postmix_userdata, &outspec, mix_buffer, work_buffer_size);
+                    MixFloat32Audio(final_mix_buffer, mix_buffer, work_buffer_size);
+                }
             }
 
-            if (((Uint8 *) mix_buffer) != device_buffer) {
+            if (((Uint8 *) final_mix_buffer) != device_buffer) {
                 // !!! FIXME: we can't promise the device buf is aligned/padded for SIMD.
-                //ConvertAudio(needed_samples * device->spec.channels, mix_buffer, SDL_AUDIO_F32, device->spec.channels, device_buffer, device->spec.format, device->spec.channels, device->work_buffer);
-                ConvertAudio(needed_samples / device->spec.channels, mix_buffer, SDL_AUDIO_F32, device->spec.channels, device->work_buffer, device->spec.format, device->spec.channels, NULL);
+                //ConvertAudio(needed_samples * device->spec.channels, final_mix_buffer, SDL_AUDIO_F32, device->spec.channels, device_buffer, device->spec.format, device->spec.channels, device->work_buffer);
+                ConvertAudio(needed_samples / device->spec.channels, final_mix_buffer, SDL_AUDIO_F32, device->spec.channels, device->work_buffer, device->spec.format, device->spec.channels, NULL);
                 SDL_memcpy(device_buffer, device->work_buffer, buffer_size);
             }
         }
@@ -837,21 +851,37 @@ SDL_bool SDL_CaptureAudioThreadIterate(SDL_AudioDevice *device)
         current_audio.impl.FlushCapture(device); // nothing wants data, dump anything pending.
     } else {
         // this SHOULD NOT BLOCK, as we are holding a lock right now. Block in WaitCaptureDevice!
-        const int rc = current_audio.impl.CaptureFromDevice(device, device->work_buffer, device->buffer_size);
-        if (rc < 0) {  // uhoh, device failed for some reason!
+        int br = current_audio.impl.CaptureFromDevice(device, device->work_buffer, device->buffer_size);
+        if (br < 0) {  // uhoh, device failed for some reason!
             retval = SDL_FALSE;
-        } else if (rc > 0) {  // queue the new data to each bound stream.
+        } else if (br > 0) {  // queue the new data to each bound stream.
             for (SDL_LogicalAudioDevice *logdev = device->logical_devices; logdev != NULL; logdev = logdev->next) {
                 if (SDL_AtomicGet(&logdev->paused)) {
                     continue;  // paused? Skip this logical device.
                 }
 
+                void *output_buffer = device->work_buffer;
+                SDL_AudioSpec outspec;
+                SDL_copyp(&outspec, &device->spec);
+
+                // I don't know why someone would want a postmix on a capture device, but we offer it for API consistency.
+                if (logdev->postmix) {
+                    // move to float format.
+                    output_buffer = device->postmix_buffer;
+                    outspec.format = SDL_AUDIO_F32;
+                    const int frames = br / SDL_AUDIO_FRAMESIZE(device->spec);
+                    br = frames * SDL_AUDIO_FRAMESIZE(outspec);
+                    ConvertAudio(frames, device->work_buffer, device->spec.format, outspec.channels, device->postmix_buffer, SDL_AUDIO_F32, outspec.channels, NULL);
+                    logdev->postmix(logdev->postmix_userdata, &outspec, device->postmix_buffer, br);
+                }
+
                 for (SDL_AudioStream *stream = logdev->bound_streams; stream != NULL; stream = stream->next_binding) {
                     /* this will hold a lock on `stream` while putting. We don't explicitly lock the streams
                        for iterating here because the binding linked list can only change while the device lock is held.
                        (we _do_ lock the stream during binding/unbinding to make sure that two threads can't try to bind
                        the same stream to different devices at the same time, though.) */
-                    if (SDL_PutAudioStreamData(stream, device->work_buffer, rc) < 0) {
+                    SDL_SetAudioStreamFormat(stream, &outspec, NULL);
+                    if (SDL_PutAudioStreamData(stream, output_buffer, br) < 0) {
                         // oh crud, we probably ran out of memory. This is possibly an overreaction to kill the audio device, but it's likely the whole thing is going down in a moment anyhow.
                         retval = SDL_FALSE;
                         break;
@@ -1138,6 +1168,9 @@ static void ClosePhysicalAudioDevice(SDL_AudioDevice *device)
     SDL_aligned_free(device->mix_buffer);
     device->mix_buffer = NULL;
 
+    SDL_aligned_free(device->postmix_buffer);
+    device->postmix_buffer = NULL;
+
     SDL_memcpy(&device->spec, &device->default_spec, sizeof (SDL_AudioSpec));
     device->sample_frames = 0;
     device->silence_value = SDL_GetSilenceValueForFormat(device->spec.format);
@@ -1395,6 +1428,28 @@ SDL_bool SDL_IsAudioDevicePaused(SDL_AudioDeviceID devid)
     return retval;
 }
 
+int SDL_SetAudioPostmixCallback(SDL_AudioDeviceID devid, SDL_AudioPostmixCallback callback, void *userdata)
+{
+    SDL_LogicalAudioDevice *logdev = ObtainLogicalAudioDevice(devid);
+    int retval = 0;
+    if (logdev) {
+        SDL_AudioDevice *device = logdev->physical_device;
+        if (!device->postmix_buffer) {
+            device->postmix_buffer = (float *)SDL_aligned_alloc(SDL_SIMDGetAlignment(), device->work_buffer_size);
+            if (device->mix_buffer == NULL) {
+                retval = SDL_OutOfMemory();
+            }
+        }
+
+        if (retval == 0) {
+            logdev->postmix = callback;
+            logdev->postmix_userdata = userdata;
+        }
+
+        SDL_UnlockMutex(device->lock);
+    }
+    return retval;
+}
 
 int SDL_BindAudioStreams(SDL_AudioDeviceID devid, SDL_AudioStream **streams, int num_streams)
 {
@@ -1782,6 +1837,14 @@ int SDL_AudioDeviceFormatChangedAlreadyLocked(SDL_AudioDevice *device, const SDL
                 kill_device = SDL_TRUE;
             }
 
+            if (device->postmix_buffer) {
+                SDL_aligned_free(device->postmix_buffer);
+                device->postmix_buffer = (float *)SDL_aligned_alloc(SDL_SIMDGetAlignment(), device->work_buffer_size);
+                if (!device->postmix_buffer) {
+                    kill_device = SDL_TRUE;
+                }
+            }
+
             SDL_aligned_free(device->mix_buffer);
             device->mix_buffer = NULL;
             if (device->spec.format != SDL_AUDIO_F32) {

src/audio/SDL_sysaudio.h

@@ -214,6 +214,12 @@ struct SDL_LogicalAudioDevice
     // SDL_TRUE if device was opened with SDL_OpenAudioDeviceStream (so it forbids binding changes, etc).
     SDL_bool simplified;
 
+    // If non-NULL, callback into the app that lets them access the final postmix buffer.
+    SDL_AudioPostmixCallback postmix;
+
+    // App-supplied pointer for postmix callback.
+    void *postmix_userdata;
+
     // double-linked list of opened devices on the same physical device.
     SDL_LogicalAudioDevice *next;
     SDL_LogicalAudioDevice *prev;
@@ -264,6 +270,7 @@ struct SDL_AudioDevice
     // Scratch buffers used for mixing.
     Uint8 *work_buffer;
     Uint8 *mix_buffer;
+    float *postmix_buffer;
 
     // Size of work_buffer (and mix_buffer) in bytes.
     int work_buffer_size;

src/dynapi/SDL_dynapi.sym

@@ -904,6 +904,7 @@ SDL3_0.0.0 {
     SDL_SetWindowFocusable;
     SDL_GetAudioStreamFrequencyRatio;
     SDL_SetAudioStreamFrequencyRatio;
+    SDL_SetAudioPostmixCallback;
     # extra symbols go here (don't modify this line)
   local: *;
 };

src/dynapi/SDL_dynapi_overrides.h

@@ -929,3 +929,4 @@
 #define SDL_SetWindowFocusable SDL_SetWindowFocusable_REAL
 #define SDL_GetAudioStreamFrequencyRatio SDL_GetAudioStreamFrequencyRatio_REAL
 #define SDL_SetAudioStreamFrequencyRatio SDL_SetAudioStreamFrequencyRatio_REAL
+#define SDL_SetAudioPostmixCallback SDL_SetAudioPostmixCallback_REAL

src/dynapi/SDL_dynapi_procs.h

@@ -975,3 +975,4 @@ SDL_DYNAPI_PROC(int,SDL_GDKGetDefaultUser,(XUserHandle *a),(a),return)
 SDL_DYNAPI_PROC(int,SDL_SetWindowFocusable,(SDL_Window *a, SDL_bool b),(a,b),return)
 SDL_DYNAPI_PROC(float,SDL_GetAudioStreamFrequencyRatio,(SDL_AudioStream *a),(a),return)
 SDL_DYNAPI_PROC(int,SDL_SetAudioStreamFrequencyRatio,(SDL_AudioStream *a, float b),(a,b),return)
+SDL_DYNAPI_PROC(int,SDL_SetAudioPostmixCallback,(SDL_AudioDeviceID a, SDL_AudioPostmixCallback b, void *c),(a,b,c),return)