[Question] How to create tracks from image and audio streams comes from socket?

[xyzbilal]

I have a project that two device communictes each other using local network and one device sends camera picture and recorded audio using flutters socket. receiver gets data and shows and plays data received from socket.
Now I want to add these datas to webrtc instance as track so that receiver transfers voice and images to webrtc peers connected.

On Sender device there will be no internet connection. Receiver will connect lo sender view Socket instance runs on Sender device. but Receiver device will connected to other socket available over internet.

What I tried are; I tried webrtc connection Sender <-> Receiver and Receiver <-> Mobile app and tried to add tracks from sender->receiver peerconnection to receiver->mobile peer connection, it no success on android platform that runs receiver codes. when I run receiver codes on linux it adds remote tracks to mobile peer video runs well but audio has problems, it sounds like robotic and has distortions so much so we cant fix audio problem. now we trying the method solution mentioned above.

wonder if I can add audio and image data streams as Local tracks to my peerconnection.
Any Idea will be appriciated.

here is my receiver code for socket solution

class ReceiverWidget extends StatefulWidget {
  const ReceiverWidget({Key? key}) : super(key: key);

  @override
  State<ReceiverWidget> createState() => _ReceiverWidgetState();
}

class _ReceiverWidgetState extends State<ReceiverWidget> with SocketServiceInterface {
  late SocketService _socket;



    RTCPeerConnection? _peerConnection;
  // Görüntü göstermek için
  Uint8List? _latestFrame;

  // Ses çalmak için
   final PlayerStream _player = PlayerStream();
   final StreamController<Uint8List> _imageStreamController =
      StreamController<Uint8List>.broadcast();


  createPC() async {
    _peerConnection = await createPeerConnection();

    //final videosource = await RTCFactory.createLocalMediaStream( 'video_stream');
  }




  @override
  void initState() {
    super.initState();
    _initSocket();
    _initPlayer();
  }

  @override
  void dispose() {
   
    _player.dispose();
    super.dispose();
  }

  void _initSocket() {
    _socket = SocketService(this, baseSocketUrl: "ws://192.168.41.37:8080", deviceId: "333");
  }


   Future<void> _initPlayer() async {
    // PlayerStream'i başlat.
    await _player.initialize(
   
    );
    await _player.start(); 
    // (Varsayılan olarak 16kHz mono PCM bekler, 
    //  Recorder tarafındaki parametrelere göre eşleştirmeniz gerekir.)
  }


  @override
  Widget build(BuildContext context) {
    return Scaffold(
      appBar: AppBar(
        title: const Text('Receiver'),
      ),
      body: Center(
        child:  StreamBuilder<Uint8List>(
          stream: _imageStreamController.stream,
          builder: (context, snapshot) {
            if (!snapshot.hasData) {
              return const Text('Henüz görüntü almadık.');
            }
            final imageData = snapshot.data!;
            return Image.memory(
              imageData,
              gaplessPlayback: true,
            );
          })
      ),
    );
  }
  
  @override
  void connect() {
    // TODO: implement connect
  }
  
  @override
  void disconnect() {
    // TODO: implement disconnect
  }
  
  @override
  void initSocket() {
    // TODO: implement initSocket
  }
  
  @override
  void onMessage(Map<String, dynamic> socketMsg) {
   
  }

 @override
  void onData(Uint8List data) {
    // Gelen veriyi buffer'a ekle


    print("Data received: ${data.length} bytes");
    _receiveBuffer.addAll(data);

    // Buffer içindeki paketleri ayıkla
    _parseBuffer();
  }


  // Marker tanımları
  final List<int> _imgStart = "<IS>".codeUnits; // Örnek: [60,73,83,62]
  final List<int> _imgEnd   = "<IE>".codeUnits; // Örnek: [60,73,69,62]
  final List<int> _audStart = "<AS>".codeUnits;
  final List<int> _audEnd   = "<AE>".codeUnits;
 List<int> _receiveBuffer = [];
  /// Buffer içindeki tüm paketleri ayıklamaya çalışır
  void _parseBuffer() {
    while (true) {
      // 1) Önce hangi tipin start marker'ı var?
      //    <IS> veya <AS> hangisi önce gelirse ona göre işlem yapalım
      final imgStartIndex = _indexOf(_receiveBuffer, _imgStart);
      final audStartIndex = _indexOf(_receiveBuffer, _audStart);

      // -1 demek yok, >= 0 demek bulundu
      final hasImgStart = (imgStartIndex >= 0);
      final hasAudStart = (audStartIndex >= 0);

      // Hangisi önce geliyor?
      int nextStartIndex;
      bool isImage;

      if (!hasImgStart && !hasAudStart) {
        // Hiç start marker yok -> parse edebileceğimiz bir şey kalmadı
        break;
      } else if (hasImgStart && hasAudStart) {
        // Her ikisi de var -> hangisi önce geliyorsa
        if (imgStartIndex < audStartIndex) {
          nextStartIndex = imgStartIndex;
          isImage = true;
        } else {
          nextStartIndex = audStartIndex;
          isImage = false;
        }
      } else if (hasImgStart) {
        // Sadece img var
        nextStartIndex = imgStartIndex;
        isImage = true;
      } else {
        // Sadece aud var
        nextStartIndex = audStartIndex;
        isImage = false;
      }

      // Start marker bulundu -> oraya kadarki veriyi atabiliriz (marker dahil)
      // Fakat, bitiş marker'ı var mı kontrol edeceğiz
      final endMarker = isImage ? _imgEnd : _audEnd;
      final endIndex = _indexOf(_receiveBuffer, endMarker, start: nextStartIndex);
      if (endIndex < 0) {
        // Bitiş marker yok, paket henüz TAM gelmemiş -> beklemeye devam
        break;
      }

      // startMarker'ın sonuna kadar atlamak için
      final startMarkerLength = isImage ? _imgStart.length : _audStart.length;
      final packetStart = nextStartIndex + startMarkerLength;
      final packetEnd = endIndex; // endMarker'ın başı

      // Paket verisi
      final length = packetEnd - packetStart;
      if (length <= 0) {
        // Geçersiz, demek ki marker'lar üst üste binmiş, vb.
        // buffer'dan marker'ları silip devam edebiliriz.
        _receiveBuffer.removeRange(nextStartIndex, endIndex + endMarker.length);
        continue;
      }

      // Bu paket verisi [packetStart, packetEnd) aralığında
      final extractedBytes = _receiveBuffer.sublist(packetStart, packetEnd);

      // Artık bu paketi işleyelim (image veya audio)
      if (isImage) {
        _handleImagePacket(Uint8List.fromList(extractedBytes));
      } else {
        _handleAudioPacket(Uint8List.fromList(extractedBytes));
      }

      // Kullanılan veriyi buffer'dan çıkar (startMarker + paket + endMarker)
      final removeEnd = endIndex + endMarker.length; 
      _receiveBuffer.removeRange(nextStartIndex, removeEnd);
    }
  }

  /// Basit "indexOf" arama fonksiyonu (List<int> içinde başka bir List<int> arar)
  int _indexOf(List<int> buffer, List<int> pattern, {int start = 0}) {
    if (pattern.isEmpty) return -1;
    for (int i = start; i <= buffer.length - pattern.length; i++) {
      bool found = true;
      for (int j = 0; j < pattern.length; j++) {
        if (buffer[i + j] != pattern[j]) {
          found = false;
          break;
        }
      }
      if (found) {
        return i;
      }
    }
    return -1;
  }



  /// Gelen görüntü verisini ekranda göstermek
  void _handleImagePacket(Uint8List data) {
    _imageStreamController.add(data);
  }

  /// Gelen ses verisini çalmak
  Future<void> _handleAudioPacket(Uint8List data) async {
    print("Audio packet received: ${data.length} bytes");
    _player.writeChunk(data);
   
  }
  
 
}