Merge pull request #159 from Jw705/feature/231127-participant-connect…

…-to-server

Feat(#58,#71,#72) 참여자 페이지 음성 스트리밍 구현, 음량 및 스피커 조절

frontend/.gitignore

@@ -22,3 +22,6 @@ dist-ssr
 *.njsproj
 *.sln
 *.sw?
+
+
+.env

frontend/package.json

@@ -10,6 +10,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
+    "dotenv": "^16.3.1",
     "fabric": "^5.3.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",

frontend/src/components/Header/components/HeaderInstructorControls.tsx

@@ -10,8 +10,9 @@ import MicOnIcon from "@/assets/svgs/micOn.svg?react";
 import MicOffIcon from "@/assets/svgs/micOff.svg?react";
 import SmallButton from "@/components/SmallButton/SmallButton";
 import Modal from "@/components/Modal/Modal";
+import { useToast } from "@/components/Toast/useToast";
 
-import selectedMicrophoneState from "./stateMicrophone";
+import selectedMicrophoneState from "./stateSelectedMicrophone";
 import micVolmeState from "./stateMicVolme";
 
 const HeaderInstructorControls = () => {
@@ -24,6 +25,7 @@ const HeaderInstructorControls = () => {
   const selectedMicrophone = useRecoilValue(selectedMicrophoneState);
   const inputMicVolume = useRecoilValue(micVolmeState);
   const setInputMicVolumeState = useSetRecoilState(micVolmeState);
+  const showToast = useToast();
 
   const timerIdRef = useRef<number | null>(null); // 경과 시간 표시 타이머 id
   const onFrameIdRef = useRef<number | null>(null); // 마이크 볼륨 측정 타이머 id
@@ -34,6 +36,18 @@ const HeaderInstructorControls = () => {
   const inputMicVolumeRef = useRef<number>(0);
   const prevInputMicVolumeRef = useRef<number>(0);
   const MEDIA_SERVER_URL = "http://localhost:3000/create-room";
+  const pc_config = {
+    iceServers: [
+      {
+        urls: ["stun:stun.l.google.com:19302"]
+      },
+      {
+        urls: import.meta.env.VITE_TURN_URL as string,
+        username: import.meta.env.VITE_TURN_USERNAME as string,
+        credential: import.meta.env.VITE_TURN_PASSWORD as string
+      }
+    ]
+  };
 
   useEffect(() => {
     inputMicVolumeRef.current = inputMicVolume;
@@ -84,7 +98,7 @@ const HeaderInstructorControls = () => {
       startTimer();
 
       // 2. 로컬 RTCPeerConnection 생성
-      pcRef.current = new RTCPeerConnection();
+      pcRef.current = new RTCPeerConnection(pc_config);
       // 3. 로컬 stream에 track 추가, 발표자의 미디어 트랙을 로컬 RTCPeerConnection에 추가
       if (updatedStreamRef.current) {
         updatedStreamRef.current.getTracks().forEach((track) => {
@@ -190,7 +204,7 @@ const HeaderInstructorControls = () => {
 
   // 경과 시간을 표시하기 위한 부분입니다
   const startTimer = () => {
-    let startTime = Date.now();
+    const startTime = Date.now();
     const updateElapsedTime = () => {
       const elapsedTime = Math.floor((Date.now() - startTime) / 1000);
       setElapsedTime(elapsedTime);
@@ -225,9 +239,11 @@ const HeaderInstructorControls = () => {
       prevInputMicVolumeRef.current = inputMicVolumeRef.current;
       setInputMicVolumeState(0);
       setIsMicOn(false);
+      showToast({ message: "마이크 음소거 되었습니다", type: "alert" });
     } else {
       setInputMicVolumeState(prevInputMicVolumeRef.current);
       setIsMicOn(true);
+      showToast({ message: "마이크 음소거가 해제되었습니다", type: "success" });
     }
   };
 

frontend/src/components/Header/components/HeaderInstructorSettingModal.tsx

@@ -1,6 +1,6 @@
 import { useState, useEffect } from "react";
 import { useSetRecoilState } from "recoil";
-import selectedMicrophoneState from "./stateMicrophone";
+import selectedMicrophoneState from "./stateSelectedMicrophone";
 import micVolmeState from "./stateMicVolme";
 
 interface HeaderSettingModalProps {
@@ -40,13 +40,13 @@ const HeaderSettingModal = ({ isSettingClicked, setIsSettingClicked }: HeaderSet
           isSettingClicked ? "opacity-100 visible" : "opacity-0 invisible"
         }`}
       >
-        <div className="flex flex-row gap-3 w-full h-10 justify-start">
+        <div className="flex flex-row gap-3 w-full justify-start">
           <p id="input-device-label">마이크 선택</p>
         </div>
 
         <select
           aria-labelledby="input-device-label"
-          className="border w-full"
+          className="border w-full rounded-xl px-3 py-4"
           onChange={(e) => setSelectedMicrophone(e.target.value)}
         >
           {microphoneDevices.map((device) => (

frontend/src/components/Header/components/HeaderParticipantControls.tsx

@@ -1,68 +1,202 @@
 import { useState, useRef, useEffect } from "react";
-// @ts-ignore
 import { useRecoilValue, useSetRecoilState } from "recoil";
-//import { io, Socket } from "socket.io-client";
+import { io, Socket } from "socket.io-client";
 import { useNavigate } from "react-router-dom";
 
 import VolumeMeter from "./VolumeMeter";
-
 import StopIcon from "@/assets/svgs/stop.svg?react";
 import MicOnIcon from "@/assets/svgs/micOn.svg?react";
 import MicOffIcon from "@/assets/svgs/micOff.svg?react";
 import SmallButton from "@/components/SmallButton/SmallButton";
 import Modal from "@/components/Modal/Modal";
+import { useToast } from "@/components/Toast/useToast";
 
-import selectedMicrophoneState from "./stateMicrophone";
-import micVolmeState from "./stateMicVolme";
+import selectedSpeakerState from "./stateSelectedSpeaker";
+import speakerVolmeState from "./stateSpeakerVolme";
 
 const HeaderParticipantControls = () => {
-  // @ts-ignore
-  const [isLectureStart, setIsLectureStart] = useState(false);
-  const [isSpeakerOn, setisSpeakerOn] = useState(true);
+  const [isSpeakerOn, setisSpeakerOn] = useState(false);
   const [isModalOpen, setIsModalOpen] = useState(false);
-  // @ts-ignore
   const [elapsedTime, setElapsedTime] = useState<number>(0);
-  // @ts-ignore
   const [micVolume, setMicVolume] = useState<number>(0);
 
-  const selectedMicrophone = useRecoilValue(selectedMicrophoneState);
-  const SpeakerVolume = useRecoilValue(micVolmeState);
-  //const setSpeakerVolumeState = useSetRecoilState(micVolmeState);
+  const [didMount, setDidMount] = useState(false);
+
+  const selectedSpeaker = useRecoilValue(selectedSpeakerState);
+  const SpeakerVolume = useRecoilValue(speakerVolmeState);
+  const setSpeakerVolume = useSetRecoilState(speakerVolmeState);
 
-  // 아래는 추후에 사용할 예정입니다.
-  //const timerIdRef = useRef<number | null>(null); // 경과 시간 표시 타이머 id
-  //const onFrameIdRef = useRef<number | null>(null); // 마이크 볼륨 측정 타이머 id
-  //const socketRef = useRef<Socket>();
-  //const pcRef = useRef<RTCPeerConnection>();
-  //const mediaStreamRef = useRef<MediaStream>();
-  //const updatedStreamRef = useRef<MediaStream>();
-  const SpeakerVolumeRef = useRef<number>(0);
-  //const prevSpeakerVolumeRef = useRef<number>(0);
+  const timerIdRef = useRef<number | null>(null); // 경과 시간 표시 타이머 id
+  const onFrameIdRef = useRef<number | null>(null); // 마이크 볼륨 측정 타이머 id
+  const socketRef = useRef<Socket>();
+  const pcRef = useRef<RTCPeerConnection>();
+  const mediaStreamRef = useRef<MediaStream>();
+  const localAudioRef = useRef<HTMLAudioElement>(null);
+  const speakerVolumeRef = useRef<number>(0);
+  const prevSpeakerVolumeRef = useRef<number>(0);
+  const audioContextRef = useRef<AudioContext | null>(null);
 
   const navigate = useNavigate();
-  //const MEDIA_SERVER_URL = "http://localhost:3000/create-room";
+  const showToast = useToast();
+  const MEDIA_SERVER_URL = "http://localhost:3000/enter-room";
+  const pc_config = {
+    iceServers: [
+      {
+        urls: ["stun:stun.l.google.com:19302"]
+      },
+      {
+        urls: import.meta.env.VITE_TURN_URL as string,
+        username: import.meta.env.VITE_TURN_USERNAME as string,
+        credential: import.meta.env.VITE_TURN_PASSWORD as string
+      }
+    ]
+  };
 
   useEffect(() => {
-    SpeakerVolumeRef.current = SpeakerVolume;
-  }, [SpeakerVolume]);
+    setDidMount(true);
+  }, []);
   useEffect(() => {
-    if (isLectureStart) {
-      // 추후 구현
+    if (didMount) {
+      enterLecture();
     }
-  }, [selectedMicrophone]);
+  }, [didMount]);
+
+  useEffect(() => {
+    speakerVolumeRef.current = SpeakerVolume;
+  }, [SpeakerVolume]);
+  useEffect(() => {
+    if (!audioContextRef.current) return;
+    (audioContextRef.current as any).setSinkId(selectedSpeaker);
+  }, [selectedSpeaker]);
+
+  const enterLecture = async () => {
+    await initConnection();
+
+    await createStudentOffer();
+    await setServerAnswer();
+    showToast({ message: "음소거 해제 후 소리를 들을 수 있습니다", type: "alert" });
+  };
 
   const leaveLecture = () => {
+    setElapsedTime(0);
+
+    if (timerIdRef.current) clearInterval(timerIdRef.current); // 경과 시간 표시 타이머 중지
+    if (onFrameIdRef.current) window.cancelAnimationFrame(onFrameIdRef.current); // 마이크 볼륨 측정 중지
+    if (socketRef.current) socketRef.current.disconnect(); // 소켓 연결 해제
+    if (pcRef.current) pcRef.current.close(); // RTCPeerConnection 해제
+    if (mediaStreamRef.current) mediaStreamRef.current.getTracks().forEach((track) => track.stop()); // 미디어 트랙 중지
+
     setIsModalOpen(false);
     navigate("/");
   };
 
+  const initConnection = async () => {
+    try {
+      socketRef.current = io(MEDIA_SERVER_URL);
+      pcRef.current = new RTCPeerConnection(pc_config);
+      const stream = new MediaStream();
+      mediaStreamRef.current = stream;
+
+      if (!pcRef.current) return;
+      pcRef.current.ontrack = (event) => {
+        if (!mediaStreamRef.current || !localAudioRef.current) return;
+        if (event.track.kind === "audio") {
+          mediaStreamRef.current.addTrack(event.track);
+          localAudioRef.current.srcObject = mediaStreamRef.current;
+        }
+      };
+    } catch (e) {
+      console.error("연결 에러", e);
+    }
+  };
+
+  function getStudentCandidate() {
+    if (!pcRef.current) return;
+    pcRef.current.onicecandidate = (e) => {
+      if (e.candidate) {
+        if (!socketRef.current) return;
+        socketRef.current.emit("studentCandidate", {
+          candidate: e.candidate,
+          studentSocketId: socketRef.current.id
+        });
+      }
+    };
+  }
+
+  async function createStudentOffer() {
+    try {
+      if (!pcRef.current || !socketRef.current) return;
+      const SDP = await pcRef.current.createOffer({
+        offerToReceiveAudio: true
+      });
+      socketRef.current.emit("studentOffer", {
+        socketId: socketRef.current.id,
+        SDP: SDP
+      });
+
+      pcRef.current.setLocalDescription(SDP);
+      getStudentCandidate();
+    } catch (e) {
+      console.log(e);
+    }
+  }
+
+  async function setServerAnswer() {
+    if (!socketRef.current) return;
+    socketRef.current.on(`${socketRef.current.id}-serverAnswer`, (data) => {
+      if (!pcRef.current) return;
+      pcRef.current.setRemoteDescription(data.SDP);
+    });
+    socketRef.current.on(`${socketRef.current.id}-serverCandidate`, (data) => {
+      if (!pcRef.current) return;
+      pcRef.current.addIceCandidate(new RTCIceCandidate(data.candidate));
+    });
+  }
+
+  const startAnalyse = () => {
+    if (!mediaStreamRef.current) return;
+    audioContextRef.current = new AudioContext();
+    const analyser = audioContextRef.current.createAnalyser();
+    const destination = audioContextRef.current.destination;
+    const mediaStreamAudioSourceNode = audioContextRef.current.createMediaStreamSource(mediaStreamRef.current);
+
+    const gainNode = audioContextRef.current.createGain();
+    mediaStreamAudioSourceNode.connect(gainNode);
+    gainNode.connect(analyser);
+    gainNode.connect(destination);
+
+    const pcmData = new Float32Array(analyser.fftSize);
+
+    const onFrame = () => {
+      gainNode.gain.value = speakerVolumeRef.current;
+      analyser.getFloatTimeDomainData(pcmData);
+      let sum = 0.0;
+      for (const amplitude of pcmData) {
+        sum += amplitude * amplitude;
+      }
+      const rms = Math.sqrt(sum / pcmData.length);
+      const normalizedVolume = Math.min(1, rms / 0.5);
+      setMicVolume(normalizedVolume);
+      onFrameIdRef.current = window.requestAnimationFrame(onFrame);
+    };
+    onFrameIdRef.current = window.requestAnimationFrame(onFrame);
+  };
+
   const mute = () => {
-    if (isSpeakerOn) {
-      // 추후 구현
+    if (!onFrameIdRef.current) {
+      // 최초 연결 후 음소거 해제
+      startAnalyse();
+      setisSpeakerOn(true);
+      showToast({ message: "음소거가 해제되었습니다", type: "success" });
+    } else if (isSpeakerOn) {
+      prevSpeakerVolumeRef.current = speakerVolumeRef.current;
+      setSpeakerVolume(0);
       setisSpeakerOn(false);
+      showToast({ message: "음소거 되었습니다", type: "alert" });
     } else {
-      // 추후 구현
+      setSpeakerVolume(prevSpeakerVolumeRef.current);
       setisSpeakerOn(true);
+      showToast({ message: "음소거가 해제되었습니다", type: "success" });
     }
   };
 
@@ -99,6 +233,7 @@ const HeaderParticipantControls = () => {
         isModalOpen={isModalOpen}
         setIsModalOpen={setIsModalOpen}
       />
+      <audio id="localAudio" playsInline autoPlay muted ref={localAudioRef}></audio>
     </>
   );
 };