Add Linux/Windows support for audio renderer (audio frame capture)

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## Unreleased
+
+* Added: Linux/Windows support for audio renderer (audio frame capture)
+
 ## 2.8.0
 
 * Added: Session API support for simpler E2EE setup

linux/CMakeLists.txt

@@ -29,6 +29,7 @@ list(APPEND PLUGIN_SOURCES
   "task_runner_linux.cc"
   "../shared_cpp/fft_processor.cpp"
   "../shared_cpp/audio_visualizer.cpp"
+  "../shared_cpp/audio_renderer.cpp"
   "../shared_cpp/pffft.c"
   "flutter/core_implementations.cc"
   "flutter/standard_codec.cc"

linux/livekit_plugin.cpp

@@ -29,10 +29,12 @@
 #include <algorithm>
 #include <cstring>
 #include <iomanip>
+#include <iostream>
 #include <map>
 #include <memory>
 #include <sstream>
 
+#include "audio_renderer.h"
 #include "audio_visualizer.h"
 
 #include "task_runner_linux.h"
@@ -159,6 +161,104 @@ class VisualizerSink : public libwebrtc::AudioTrackSink {
   int bar_count_ = 7;
 };
 
+class AudioRendererSink : public libwebrtc::AudioTrackSink {
+public:
+  AudioRendererSink(
+      BinaryMessenger *messenger, std::string event_channel_name,
+      libwebrtc::scoped_refptr<libwebrtc::RTCMediaTrack> media_track,
+      RendererAudioFormat format)
+      : channel_(
+            std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
+                messenger, event_channel_name,
+                &flutter::StandardMethodCodec::GetInstance())),
+        media_track_(media_track), format_(format) {
+    task_runner_ = std::make_unique<livekit_client_plugin::TaskRunnerLinux>();
+    auto handler = std::make_unique<
+        flutter::StreamHandlerFunctions<flutter::EncodableValue>>(
+        [&](const flutter::EncodableValue *arguments,
+            std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>
+                &&events)
+            -> std::unique_ptr<
+                flutter::StreamHandlerError<flutter::EncodableValue>> {
+          sink_ = std::move(events);
+          on_listen_called_ = true;
+          return nullptr;
+        },
+        [&](const flutter::EncodableValue *arguments)
+            -> std::unique_ptr<
+                flutter::StreamHandlerError<flutter::EncodableValue>> {
+          on_listen_called_ = false;
+          return nullptr;
+        });
+
+    channel_->SetStreamHandler(std::move(handler));
+    ((libwebrtc::RTCAudioTrack *)media_track_.get())->AddSink(this);
+  }
+  ~AudioRendererSink() override {}
+
+public:
+  void OnData(const void *audio_data, int bits_per_sample, int sample_rate,
+              size_t number_of_channels, size_t number_of_frames) override {
+    // Unlike VisualizerSink, live audio must not be queued while no
+    // subscriber is attached — drop the frame instead.
+    if (!on_listen_called_) {
+      return;
+    }
+
+    AudioConversionResult conversion =
+        ConvertAudioData(audio_data, bits_per_sample, sample_rate,
+                        number_of_channels, number_of_frames, format_);
+    if (!conversion.success) {
+      LogDroppedFrame();
+      return;
+    }
+
+    EncodableMap payload;
+    payload[EncodableValue("commonFormat")] =
+        EncodableValue(format_.common_format);
+    payload[EncodableValue("sampleRate")] =
+        EncodableValue(format_.sample_rate);
+    payload[EncodableValue("channels")] = EncodableValue(conversion.channels);
+    payload[EncodableValue("frameLength")] =
+        EncodableValue(conversion.frame_length);
+    payload[EncodableValue("data")] = EncodableValue(std::move(conversion.data));
+
+    PostEvent(EncodableValue(payload));
+  }
+
+  void PostEvent(const flutter::EncodableValue &event) {
+    std::weak_ptr<flutter::EventSink<EncodableValue>> weak_sink = sink_;
+    task_runner_->EnqueueTask([weak_sink, event]() {
+      auto sink = weak_sink.lock();
+      if (sink) {
+        sink->Success(event);
+      }
+    });
+  }
+
+  void RemoveSink() {
+    ((libwebrtc::RTCAudioTrack *)media_track_.get())->RemoveSink(this);
+    channel_->SetStreamHandler(nullptr);
+  }
+
+private:
+  void LogDroppedFrame() {
+    dropped_frame_count_ += 1;
+    if (dropped_frame_count_ <= 5 || dropped_frame_count_ % 100 == 0) {
+      std::cerr << "[LiveKit] Dropping audio frame #" << dropped_frame_count_
+                << " for audio renderer" << std::endl;
+    }
+  }
+
+  std::unique_ptr<livekit_client_plugin::TaskRunnerLinux> task_runner_;
+  std::unique_ptr<flutter::EventChannel<flutter::EncodableValue>> channel_;
+  std::shared_ptr<flutter::EventSink<flutter::EncodableValue>> sink_;
+  bool on_listen_called_ = false;
+  libwebrtc::scoped_refptr<libwebrtc::RTCMediaTrack> media_track_;
+  RendererAudioFormat format_;
+  int64_t dropped_frame_count_ = 0;
+};
+
 class LiveKitPlugin : public flutter::Plugin {
 public:
   static void RegisterWithRegistrar(flutter::PluginRegistrar *registrar);
@@ -176,6 +276,8 @@ class LiveKitPlugin : public flutter::Plugin {
 private:
   flutter_webrtc_plugin::FlutterWebRTC *webrtc_instance_ = nullptr;
   std::unordered_map<std::string, std::unique_ptr<VisualizerSink>> visualizers_;
+  std::unordered_map<std::string, std::unique_ptr<AudioRendererSink>>
+      renderers_;
   BinaryMessenger *messenger_ = nullptr;
   mutable std::mutex mutex_;
 };
@@ -274,6 +376,82 @@ void LiveKitPlugin::HandleMethodCall(
     }
 
     result->Success();
+  } else if (method_call.method_name().compare("startAudioRenderer") == 0) {
+    if (!method_call.arguments()) {
+      result->Error("Bad Arguments", "Null arguments received");
+      return;
+    }
+    flutter::EncodableMap params =
+        GetValue<flutter::EncodableMap>(*method_call.arguments());
+    std::string trackId = findString(params, "trackId");
+    std::string rendererId = findString(params, "rendererId");
+    if (trackId.empty()) {
+      result->Error("INVALID_ARGUMENT", "trackId is required");
+      return;
+    }
+    if (rendererId.empty()) {
+      result->Error("INVALID_ARGUMENT", "rendererId is required");
+      return;
+    }
+
+    auto formatIt = params.find(EncodableValue("format"));
+    if (formatIt == params.end()) {
+      result->Error("INVALID_ARGUMENT", "format is required");
+      return;
+    }
+    if (!TypeIs<EncodableMap>(formatIt->second)) {
+      result->Error("INVALID_ARGUMENT", "Failed to parse format");
+      return;
+    }
+    EncodableMap formatMap = GetValue<EncodableMap>(formatIt->second);
+    RendererAudioFormat format = RendererAudioFormat::FromValues(
+        findString(formatMap, "commonFormat"), findInt(formatMap, "sampleRate"),
+        findInt(formatMap, "channels"));
+
+    libwebrtc::scoped_refptr<libwebrtc::RTCMediaTrack> media_track =
+        webrtc_instance_->MediaTrackForId(trackId);
+    if (!media_track) {
+      result->Error("INVALID_ARGUMENT", "No such track");
+      return;
+    }
+
+    mutex_.lock();
+    if (renderers_.find(rendererId) != renderers_.end()) {
+      mutex_.unlock();
+      result->Success(flutter::EncodableValue(true));
+      return;
+    }
+
+    std::ostringstream oss;
+    oss << "io.livekit.audio.renderer/channel-" << rendererId;
+
+    renderers_[rendererId] = std::make_unique<AudioRendererSink>(
+        messenger_, oss.str(), media_track, format);
+    mutex_.unlock();
+
+    result->Success(flutter::EncodableValue(true));
+  } else if (method_call.method_name().compare("stopAudioRenderer") == 0) {
+    if (!method_call.arguments()) {
+      result->Error("Bad Arguments", "Null arguments received");
+      return;
+    }
+    flutter::EncodableMap params =
+        GetValue<flutter::EncodableMap>(*method_call.arguments());
+    std::string rendererId = findString(params, "rendererId");
+    if (rendererId.empty()) {
+      result->Error("INVALID_ARGUMENT", "rendererId is required");
+      return;
+    }
+
+    mutex_.lock();
+    auto it = renderers_.find(rendererId);
+    if (it != renderers_.end()) {
+      it->second->RemoveSink();
+      renderers_.erase(it);
+    }
+    mutex_.unlock();
+
+    result->Success(flutter::EncodableValue(true));
   } else {
     result->NotImplemented();
   }

shared_cpp/audio_renderer.cpp

@@ -0,0 +1,160 @@
+#include "audio_renderer.h"
+
+#include <algorithm>
+#include <cstring>
+
+namespace {
+
+std::vector<int16_t> UpsampleAudio(const int16_t *src, int src_frames,
+                                   int out_frames, int channels) {
+  std::vector<int16_t> out(static_cast<size_t>(out_frames) * channels);
+
+  // Edge case: a single source frame — just repeat it.
+  if (src_frames <= 1) {
+    for (int f = 0; f < out_frames; ++f) {
+      for (int ch = 0; ch < channels; ++ch) {
+        out[f * channels + ch] = src[ch];
+      }
+    }
+    return out;
+  }
+
+  double ratio =
+      static_cast<double>(src_frames) / static_cast<double>(out_frames);
+
+  for (int f = 0; f < out_frames; ++f) {
+    double src_pos = f * ratio;
+    int idx = std::min(static_cast<int>(src_pos), src_frames - 2);
+    float frac = static_cast<float>(src_pos - idx);
+
+    for (int ch = 0; ch < channels; ++ch) {
+      int16_t s0 = src[idx * channels + ch];
+      int16_t s1 = src[(idx + 1) * channels + ch];
+      int sample = static_cast<int>(s0 + frac * (s1 - s0));
+      sample = std::clamp(sample, static_cast<int>(INT16_MIN),
+                          static_cast<int>(INT16_MAX));
+      out[f * channels + ch] = static_cast<int16_t>(sample);
+    }
+  }
+
+  return out;
+}
+
+std::vector<int16_t> DownsampleAudio(const int16_t *src, int src_frames,
+                                     int out_frames, int src_rate,
+                                     int target_rate, int channels) {
+  std::vector<int16_t> out(static_cast<size_t>(out_frames) * channels);
+  double ratio =
+      static_cast<double>(src_rate) / static_cast<double>(target_rate);
+
+  for (int f = 0; f < out_frames; ++f) {
+    int src_start = static_cast<int>(f * ratio);
+    int src_end = std::min(static_cast<int>((f + 1) * ratio), src_frames);
+
+    for (int ch = 0; ch < channels; ++ch) {
+      int64_t sum = 0;
+      for (int i = src_start; i < src_end; ++i) {
+        sum += src[i * channels + ch];
+      }
+      int count = src_end - src_start;
+      out[f * channels + ch] =
+          count > 0 ? static_cast<int16_t>(std::clamp<int64_t>(
+                          sum / count, INT16_MIN, INT16_MAX))
+                    : 0;
+    }
+  }
+
+  return out;
+}
+
+} // namespace
+
+RendererAudioFormat RendererAudioFormat::FromValues(
+    const std::string &common_format, int sample_rate, int channels) {
+  RendererAudioFormat format;
+  format.common_format = common_format.empty() ? "int16" : common_format;
+  format.sample_rate = sample_rate > 0 ? sample_rate : 48000;
+  format.channels = channels > 0 ? channels : 1;
+  return format;
+}
+
+std::vector<int16_t> ResampleAudio(const int16_t *src, int src_frames,
+                                   int src_rate, int target_rate,
+                                   int channels, int &out_frames) {
+  if (src_rate == target_rate || src_frames <= 0 || channels <= 0) {
+    out_frames = src_frames;
+    return std::vector<int16_t>(
+        src, src + static_cast<size_t>(std::max(src_frames, 0)) * channels);
+  }
+
+  out_frames = static_cast<int>((static_cast<int64_t>(src_frames) *
+                                 target_rate) /
+                                src_rate);
+  if (out_frames <= 0) {
+    out_frames = 0;
+    return {};
+  }
+
+  if (target_rate > src_rate) {
+    return UpsampleAudio(src, src_frames, out_frames, channels);
+  }
+  return DownsampleAudio(src, src_frames, out_frames, src_rate, target_rate,
+                         channels);
+}
+
+AudioConversionResult
+ConvertAudioData(const void *audio_data, int bits_per_sample, int sample_rate,
+                 size_t number_of_channels, size_t number_of_frames,
+                 const RendererAudioFormat &target_format) {
+  AudioConversionResult result;
+
+  // WebRTC AudioTrackSink always delivers 16-bit signed int16 PCM.
+  if (bits_per_sample != 16 || number_of_channels == 0 ||
+      number_of_frames == 0) {
+    return result;
+  }
+
+  int channels = static_cast<int>(number_of_channels);
+  int src_frames = static_cast<int>(number_of_frames);
+  const int16_t *src = reinterpret_cast<const int16_t *>(audio_data);
+
+  int out_frames = 0;
+  std::vector<int16_t> resampled = ResampleAudio(
+      src, src_frames, sample_rate, target_format.sample_rate, channels,
+      out_frames);
+  if (out_frames <= 0) {
+    return result;
+  }
+
+  int requested_channels = std::max(target_format.channels, 1);
+  int out_channels = std::min(requested_channels, channels);
+
+  result.frame_length = out_frames;
+  result.channels = out_channels;
+
+  if (target_format.common_format == "float32") {
+    result.data.resize(static_cast<size_t>(out_frames) * out_channels * 4);
+    for (int f = 0; f < out_frames; ++f) {
+      for (int ch = 0; ch < out_channels; ++ch) {
+        float sample = resampled[f * channels + ch] / 32767.0f;
+        size_t offset = (static_cast<size_t>(f) * out_channels + ch) * 4;
+        // memcpy relies on the host being little-endian, true for the
+        // x86/x64/ARM desktop targets this plugin builds for.
+        std::memcpy(&result.data[offset], &sample, sizeof(float));
+      }
+    }
+  } else {
+    result.data.resize(static_cast<size_t>(out_frames) * out_channels * 2);
+    for (int f = 0; f < out_frames; ++f) {
+      for (int ch = 0; ch < out_channels; ++ch) {
+        int16_t sample = resampled[f * channels + ch];
+        size_t offset = (static_cast<size_t>(f) * out_channels + ch) * 2;
+        result.data[offset] = static_cast<uint8_t>(sample & 0xFF);
+        result.data[offset + 1] = static_cast<uint8_t>((sample >> 8) & 0xFF);
+      }
+    }
+  }
+
+  result.success = true;
+  return result;
+}