Created the InteractiveSpeaking exercise

src/components/Exercises/FillBlanks.tsx

@@ -130,6 +130,7 @@ export default function FillBlanks({
 			<div className="flex flex-col gap-4 mt-4 h-full mb-20">
 				{(!!currentBlankId || isDrawerShowing) && (
 					<WordsDrawer
+						key={currentBlankId}
 						blankId={currentBlankId}
 						words={words.map((word) => ({word, isDisabled: allowRepetition ? false : answers.map((x) => x.solution).includes(word)}))}
 						previouslySelectedWord={currentBlankId ? answers.find((x) => x.id === currentBlankId)?.solution : undefined}

src/components/Exercises/InteractiveSpeaking.tsx

@@ -0,0 +1,240 @@
+import {InteractiveSpeakingExercise} from "@/interfaces/exam";
+import {CommonProps} from ".";
+import {useEffect, useState} from "react";
+import {BsCheckCircleFill, BsMicFill, BsPauseCircle, BsPlayCircle, BsTrashFill} from "react-icons/bs";
+import dynamic from "next/dynamic";
+import Button from "../Low/Button";
+import useExamStore from "@/stores/examStore";
+
+const Waveform = dynamic(() => import("../Waveform"), {ssr: false});
+const ReactMediaRecorder = dynamic(() => import("react-media-recorder").then((mod) => mod.ReactMediaRecorder), {
+	ssr: false,
+});
+
+export default function InteractiveSpeaking({id, title, text, type, prompts, onNext, onBack}: InteractiveSpeakingExercise & CommonProps) {
+	const [recordingDuration, setRecordingDuration] = useState(0);
+	const [isRecording, setIsRecording] = useState(false);
+	const [mediaBlob, setMediaBlob] = useState<string>();
+	const [promptIndex, setPromptIndex] = useState(0);
+	const [answers, setAnswers] = useState<{prompt: string; blob: string}[]>([]);
+
+	const hasExamEnded = useExamStore((state) => state.hasExamEnded);
+
+	useEffect(() => {
+		if (hasExamEnded) {
+			onNext({
+				exercise: id,
+				solutions: mediaBlob ? [{id, solution: mediaBlob}] : [],
+				score: {correct: 1, total: 1, missing: 0},
+				type,
+			});
+		}
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+	}, [hasExamEnded]);
+
+	useEffect(() => {
+		let recordingInterval: NodeJS.Timer | undefined = undefined;
+		if (isRecording) {
+			recordingInterval = setInterval(() => setRecordingDuration((prev) => prev + 1), 1000);
+		} else if (recordingInterval) {
+			clearInterval(recordingInterval);
+		}
+
+		return () => {
+			if (recordingInterval) clearInterval(recordingInterval);
+		};
+	}, [isRecording]);
+
+	useEffect(() => {
+		if (promptIndex === answers.length - 1) {
+			setMediaBlob(answers[promptIndex].blob);
+		}
+	}, [answers, promptIndex]);
+
+	const saveAnswer = () => {
+		const answer = {
+			prompt: prompts[promptIndex].text,
+			blob: mediaBlob!,
+		};
+
+		setAnswers((prev) => [...prev, answer]);
+		setMediaBlob(undefined);
+	};
+
+	return (
+		<div className="flex flex-col h-full w-full gap-9">
+			<div className="flex flex-col w-full gap-8 bg-mti-gray-smoke rounded-xl py-8 pb-12 px-16">
+				<div className="flex flex-col gap-3">
+					<span className="font-semibold">{title}</span>
+				</div>
+				{prompts && prompts.length > 0 && (
+					<div className="flex flex-col gap-4 w-full items-center">
+						<video key={promptIndex} autoPlay controls className="max-w-3xl rounded-xl">
+							<source src={prompts[promptIndex].video_url} />
+						</video>
+					</div>
+				)}
+			</div>
+
+			<ReactMediaRecorder
+				audio
+				key={promptIndex}
+				onStop={(blob) => setMediaBlob(blob)}
+				render={({status, startRecording, stopRecording, pauseRecording, resumeRecording, clearBlobUrl, mediaBlobUrl}) => (
+					<div className="w-full p-4 px-8 bg-transparent border-2 border-mti-gray-platinum rounded-2xl flex-col gap-8 items-center">
+						<p className="text-base font-normal">Record your answer:</p>
+						<div className="flex gap-8 items-center justify-center py-8">
+							{status === "idle" && (
+								<>
+									<div className="w-full h-2 max-w-4xl bg-mti-gray-smoke rounded-full" />
+									{status === "idle" && (
+										<BsMicFill
+											onClick={() => {
+												setRecordingDuration(0);
+												startRecording();
+												setIsRecording(true);
+											}}
+											className="h-5 w-5 text-mti-gray-cool cursor-pointer"
+										/>
+									)}
+								</>
+							)}
+							{status === "recording" && (
+								<>
+									<div className="flex gap-4 items-center">
+										<span className="text-xs w-9">
+											{Math.floor(recordingDuration / 60)
+												.toString(10)
+												.padStart(2, "0")}
+											:
+											{Math.floor(recordingDuration % 60)
+												.toString(10)
+												.padStart(2, "0")}
+										</span>
+									</div>
+									<div className="w-full h-2 max-w-4xl bg-mti-gray-smoke rounded-full" />
+									<div className="flex gap-4 items-center">
+										<BsPauseCircle
+											onClick={() => {
+												setIsRecording(false);
+												pauseRecording();
+											}}
+											className="text-red-500 w-8 h-8 cursor-pointer"
+										/>
+										<BsCheckCircleFill
+											onClick={() => {
+												setIsRecording(false);
+												stopRecording();
+											}}
+											className="text-mti-purple-light w-8 h-8 cursor-pointer"
+										/>
+									</div>
+								</>
+							)}
+							{status === "paused" && (
+								<>
+									<div className="flex gap-4 items-center">
+										<span className="text-xs w-9">
+											{Math.floor(recordingDuration / 60)
+												.toString(10)
+												.padStart(2, "0")}
+											:
+											{Math.floor(recordingDuration % 60)
+												.toString(10)
+												.padStart(2, "0")}
+										</span>
+									</div>
+									<div className="w-full h-2 max-w-4xl bg-mti-gray-smoke rounded-full" />
+									<div className="flex gap-4 items-center">
+										<BsPlayCircle
+											onClick={() => {
+												setIsRecording(true);
+												resumeRecording();
+											}}
+											className="text-mti-purple-light w-8 h-8 cursor-pointer"
+										/>
+										<BsCheckCircleFill
+											onClick={() => {
+												setIsRecording(false);
+												stopRecording();
+											}}
+											className="text-mti-purple-light w-8 h-8 cursor-pointer"
+										/>
+									</div>
+								</>
+							)}
+							{status === "stopped" && mediaBlobUrl && (
+								<>
+									<Waveform audio={mediaBlobUrl} waveColor="#FCDDEC" progressColor="#EF5DA8" />
+									<div className="flex gap-4 items-center">
+										<BsTrashFill
+											className="text-mti-gray-cool cursor-pointer w-5 h-5"
+											onClick={() => {
+												setRecordingDuration(0);
+												clearBlobUrl();
+												setMediaBlob(undefined);
+											}}
+										/>
+
+										<BsMicFill
+											onClick={() => {
+												clearBlobUrl();
+												setRecordingDuration(0);
+												startRecording();
+												setIsRecording(true);
+												setMediaBlob(undefined);
+											}}
+											className="h-5 w-5 text-mti-gray-cool cursor-pointer"
+										/>
+									</div>
+								</>
+							)}
+						</div>
+					</div>
+				)}
+			/>
+
+			<div className="self-end flex justify-between w-full  gap-8">
+				<Button
+					color="purple"
+					variant="outline"
+					onClick={() =>
+						onBack({
+							exercise: id,
+							solutions: answers,
+							score: {correct: 1, total: 1, missing: 0},
+							type,
+						})
+					}
+					className="max-w-[200px] self-end w-full">
+					Back
+				</Button>
+				<Button
+					color="purple"
+					disabled={!mediaBlob}
+					onClick={() => {
+						saveAnswer();
+						if (promptIndex + 1 < prompts.length) {
+							setPromptIndex((prev) => prev + 1);
+							return;
+						}
+						onNext({
+							exercise: id,
+							solutions: [
+								...answers,
+								{
+									prompt: prompts[promptIndex].text,
+									blob: mediaBlob!,
+								},
+							],
+							score: {correct: 1, total: 1, missing: 0},
+							type,
+						});
+					}}
+					className="max-w-[200px] self-end w-full">
+					{promptIndex + 1 < prompts.length ? "Next Prompt" : "Submit"}
+				</Button>
+			</div>
+		</div>
+	);
+}

src/components/Exercises/index.tsx

@@ -1,6 +1,7 @@
 import {
 	Exercise,
 	FillBlanksExercise,
+	InteractiveSpeakingExercise,
 	MatchSentencesExercise,
 	MultipleChoiceExercise,
 	SpeakingExercise,
@@ -16,6 +17,7 @@ import WriteBlanks from "./WriteBlanks";
 import Writing from "./Writing";
 import Speaking from "./Speaking";
 import TrueFalse from "./TrueFalse";
+import InteractiveSpeaking from "./InteractiveSpeaking";
 
 const MatchSentences = dynamic(() => import("@/components/Exercises/MatchSentences"), {ssr: false});
 
@@ -40,5 +42,7 @@ export const renderExercise = (exercise: Exercise, onNext: (userSolutions: UserS
 			return <Writing key={exercise.id} {...(exercise as WritingExercise)} onNext={onNext} onBack={onBack} />;
 		case "speaking":
 			return <Speaking key={exercise.id} {...(exercise as SpeakingExercise)} onNext={onNext} onBack={onBack} />;
+		case "interactiveSpeaking":
+			return <InteractiveSpeaking key={exercise.id} {...(exercise as InteractiveSpeakingExercise)} onNext={onNext} onBack={onBack} />;
 	}
 };

src/components/Solutions/InteractiveSpeaking.tsx

@@ -0,0 +1,124 @@
+/* eslint-disable @next/next/no-img-element */
+import {InteractiveSpeakingExercise} from "@/interfaces/exam";
+import {CommonProps} from ".";
+import {useEffect, useState} from "react";
+import Button from "../Low/Button";
+import dynamic from "next/dynamic";
+import axios from "axios";
+import {speakingReverseMarking} from "@/utils/score";
+
+const Waveform = dynamic(() => import("../Waveform"), {ssr: false});
+
+export default function InteractiveSpeaking({
+	id,
+	type,
+	title,
+	text,
+	prompts,
+	userSolutions,
+	onNext,
+	onBack,
+}: InteractiveSpeakingExercise & CommonProps) {
+	const [solutionsURL, setSolutionsURL] = useState<string[]>([]);
+
+	useEffect(() => {
+		if (userSolutions && userSolutions.length > 0) {
+			Promise.all(userSolutions[0].solution.map((x) => axios.post(`/api/speaking`, {path: x.answer}, {responseType: "arraybuffer"}))).then(
+				(values) => {
+					setSolutionsURL(
+						values.map(({data}) => {
+							const blob = new Blob([data], {type: "audio/wav"});
+							const url = URL.createObjectURL(blob);
+
+							return url;
+						}),
+					);
+				},
+			);
+		}
+	}, [userSolutions]);
+
+	return (
+		<>
+			<div className="flex flex-col h-full w-full gap-8 mb-20">
+				<div className="flex flex-col w-full gap-8 bg-mti-gray-smoke rounded-xl py-8 pb-12 px-16">
+					<div className="flex flex-col gap-3">
+						<span className="font-semibold">{title}</span>
+					</div>
+					<div className="flex flex-col gap-4">
+						<span className="font-bold">You should talk about the following things:</span>
+						<div className="grid grid-cols-3 gap-6 text-center">
+							{prompts.map((x, index) => (
+								<div className="italic flex flex-col gap-2 text-sm" key={index}>
+									<video key={index} controls className="">
+										<source src={x.video_url} />
+									</video>
+									<span>{x.text}</span>
+								</div>
+							))}
+						</div>
+					</div>
+				</div>
+
+				<div className="w-full h-full flex flex-col gap-8">
+					<div className="flex items-center gap-8">
+						{solutionsURL.map((x, index) => (
+							<div
+								key={index}
+								className="w-full min-w-[460px] p-4 px-8 bg-transparent border-2 border-mti-gray-platinum rounded-2xl flex-col gap-8 items-center">
+								<div className="flex gap-8 items-center justify-center py-8">
+									<Waveform audio={x} waveColor="#FCDDEC" progressColor="#EF5DA8" />
+								</div>
+							</div>
+						))}
+					</div>
+
+					{userSolutions && userSolutions.length > 0 && (
+						<div className="flex flex-col gap-4 w-full">
+							<div className="flex gap-4 px-1">
+								{Object.keys(userSolutions[0].evaluation!.task_response).map((key) => (
+									<div className="bg-ielts-speaking text-ielts-speaking-light rounded-xl px-4 py-2" key={key}>
+										{key}: Level {userSolutions[0].evaluation!.task_response[key]}
+									</div>
+								))}
+							</div>
+							<div className="w-full h-full min-h-fit cursor-text px-7 py-8 bg-mti-gray-smoke rounded-3xl">
+								{userSolutions[0].evaluation!.comment}
+							</div>
+						</div>
+					)}
+				</div>
+			</div>
+
+			<div className="self-end flex justify-between w-full gap-8 absolute bottom-8 left-0 px-8">
+				<Button
+					color="purple"
+					variant="outline"
+					onClick={() =>
+						onBack({
+							exercise: id,
+							solutions: userSolutions,
+							score: {total: 100, missing: 0, correct: speakingReverseMarking[userSolutions[0]!.evaluation!.overall] || 0},
+							type,
+						})
+					}
+					className="max-w-[200px] self-end w-full">
+					Back
+				</Button>
+				<Button
+					color="purple"
+					onClick={() =>
+						onNext({
+							exercise: id,
+							solutions: userSolutions,
+							score: {total: 100, missing: 0, correct: speakingReverseMarking[userSolutions[0]!.evaluation!.overall] || 0},
+							type,
+						})
+					}
+					className="max-w-[200px] self-end w-full">
+					Next
+				</Button>
+			</div>
+		</>
+	);
+}

src/components/Solutions/index.tsx

@@ -1,6 +1,7 @@
 import {
 	Exercise,
 	FillBlanksExercise,
+	InteractiveSpeakingExercise,
 	MatchSentencesExercise,
 	MultipleChoiceExercise,
 	SpeakingExercise,
@@ -11,6 +12,7 @@ import {
 } from "@/interfaces/exam";
 import dynamic from "next/dynamic";
 import FillBlanks from "./FillBlanks";
+import InteractiveSpeaking from "./InteractiveSpeaking";
 import MultipleChoice from "./MultipleChoice";
 import Speaking from "./Speaking";
 import TrueFalseSolution from "./TrueFalse";
@@ -25,6 +27,8 @@ export interface CommonProps {
 }
 
 export const renderSolution = (exercise: Exercise, onNext: () => void, onBack: () => void) => {
+	console.log(exercise);
+
 	switch (exercise.type) {
 		case "fillBlanks":
 			return <FillBlanks {...(exercise as FillBlanksExercise)} onNext={onNext} onBack={onBack} />;
@@ -40,5 +44,7 @@ export const renderSolution = (exercise: Exercise, onNext: () => void, onBack: (
 			return <Writing {...(exercise as WritingExercise)} onNext={onNext} onBack={onBack} />;
 		case "speaking":
 			return <Speaking {...(exercise as SpeakingExercise)} onNext={onNext} onBack={onBack} />;
+		case "interactiveSpeaking":
+			return <InteractiveSpeaking {...(exercise as InteractiveSpeakingExercise)} onNext={onNext} onBack={onBack} />;
 	}
 };

src/interfaces/exam.ts

@@ -68,7 +68,8 @@ export type Exercise =
 	| MultipleChoiceExercise
 	| WriteBlanksExercise
 	| WritingExercise
-	| SpeakingExercise;
+	| SpeakingExercise
+	| InteractiveSpeakingExercise;
 
 export interface Evaluation {
 	comment: string;
@@ -108,13 +109,13 @@ export interface SpeakingExercise {
 
 export interface InteractiveSpeakingExercise {
 	id: string;
-	type: "speaking";
+	type: "interactiveSpeaking";
 	title: string;
 	text: string;
 	prompts: {text: string; video_url: string}[];
 	userSolutions: {
 		id: string;
-		solution: string;
+		solution: {question: string; answer: string}[];
 		evaluation?: Evaluation;
 	}[];
 }

src/pages/api/evaluate/interactiveSpeaking.ts

@@ -0,0 +1,59 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type {NextApiRequest, NextApiResponse} from "next";
+import {withIronSessionApiRoute} from "iron-session/next";
+import {sessionOptions} from "@/lib/session";
+import axios from "axios";
+import formidable from "formidable-serverless";
+import {getStorage, ref, uploadBytes} from "firebase/storage";
+import fs from "fs";
+import {app} from "@/firebase";
+
+export default withIronSessionApiRoute(handler, sessionOptions);
+
+async function handler(req: NextApiRequest, res: NextApiResponse) {
+	if (!req.session.user) {
+		res.status(401).json({ok: false});
+		return;
+	}
+
+	const storage = getStorage(app);
+
+	const form = formidable({keepExtensions: true});
+	await form.parse(req, async (err: any, fields: any, files: any) => {
+		if (err) console.log(err);
+
+		const uploadingAudios = await Promise.all(
+			Object.keys(files).map(async (fileID: string) => {
+				const audioFile = files[fileID];
+				const questionID = fileID.replace("answer_", "question_");
+
+				const audioFileRef = ref(storage, `speaking_recordings/${(audioFile as any).path.replace("upload_", "")}`);
+
+				const binary = fs.readFileSync((audioFile as any).path).buffer;
+				const snapshot = await uploadBytes(audioFileRef, binary);
+
+				fs.rmSync((audioFile as any).path);
+
+				return {question: fields[questionID], answer: snapshot.metadata.fullPath};
+			}),
+		);
+
+		const backendRequest = await axios.post(
+			`${process.env.BACKEND_URL}/speaking_task_3`,
+			{answers: uploadingAudios},
+			{
+				headers: {
+					Authorization: `Bearer ${process.env.BACKEND_JWT}`,
+				},
+			},
+		);
+
+		res.status(200).json({...backendRequest.data, answer: uploadingAudios});
+	});
+}
+
+export const config = {
+	api: {
+		bodyParser: false,
+	},
+};

src/pages/exercises.tsx

@@ -71,7 +71,7 @@ export default function Page() {
 
 	const {user} = useUser({redirectTo: "/login"});
 
-	useEffect(() => console.log({examId: exam?.id}), [exam]);
+	useEffect(() => console.log({examId: exam?.id, exam}), [exam]);
 	useEffect(() => setSessionId(uuidv4()), []);
 
 	useEffect(() => {

src/utils/evaluation.ts

@@ -1,4 +1,4 @@
-import {Evaluation, Exam, SpeakingExercise, UserSolution, WritingExercise} from "@/interfaces/exam";
+import {Evaluation, Exam, InteractiveSpeakingExercise, SpeakingExercise, UserSolution, WritingExercise} from "@/interfaces/exam";
 import axios from "axios";
 import {speakingReverseMarking, writingReverseMarking} from "./score";
 
@@ -30,16 +30,23 @@ export const evaluateSpeakingAnswer = async (exams: Exam[], examId: string, exer
 	const speakingExam = exams.find((x) => x.id === examId)!;
 	const exercise = speakingExam.exercises.find((x) => x.id === exerciseId);
 
-	if (exercise?.type === "speaking") {
-		return await evaluateSpeakingExercise(exercise, exerciseId, solution);
+	switch (exercise?.type) {
+		case "speaking":
+			return await evaluateSpeakingExercise(exercise, exerciseId, solution);
+		case "interactiveSpeaking":
+			return await evaluateInteractiveSpeakingExercise(exerciseId, solution);
+		default:
+			return undefined;
 	}
+};
 
-	return undefined;
+const downloadBlob = async (url: string): Promise<Buffer> => {
+	const blobResponse = await axios.get(url, {responseType: "arraybuffer"});
+	return Buffer.from(blobResponse.data, "binary");
 };
 
 const evaluateSpeakingExercise = async (exercise: SpeakingExercise, exerciseId: string, solution: UserSolution) => {
-	const blobResponse = await axios.get(solution.solutions[0].solution.trim(), {responseType: "arraybuffer"});
-	const audioBlob = Buffer.from(blobResponse.data, "binary");
+	const audioBlob = await downloadBlob(solution.solutions[0].solution.trim());
 	const audioFile = new File([audioBlob], "audio.wav", {type: "audio/wav"});
 
 	const formData = new FormData();
@@ -68,3 +75,43 @@ const evaluateSpeakingExercise = async (exercise: SpeakingExercise, exerciseId:
 
 	return undefined;
 };
+
+const evaluateInteractiveSpeakingExercise = async (exerciseId: string, solution: UserSolution) => {
+	const promiseParts = solution.solutions.map(async (x: {prompt: string; blob: string}) => ({
+		question: x.prompt,
+		answer: await downloadBlob(x.blob),
+	}));
+	const body = await Promise.all(promiseParts);
+
+	const formData = new FormData();
+	body.forEach(({question, answer}) => {
+		const seed = Math.random().toString().replace("0.", "");
+
+		const audioFile = new File([answer], `${seed}.wav`, {type: "audio/wav"});
+
+		formData.append(`question_${seed}`, question);
+		formData.append(`answer_${seed}`, audioFile, `${seed}.wav`);
+	});
+
+	const config = {
+		headers: {
+			"Content-Type": "audio/mp3",
+		},
+	};
+
+	const response = await axios.post("/api/evaluate/interactiveSpeaking", formData, config);
+
+	if (response.status === 200) {
+		return {
+			...solution,
+			score: {
+				correct: speakingReverseMarking[response.data.overall] || 0,
+				missing: 0,
+				total: 100,
+			},
+			solutions: [{id: exerciseId, solution: response.data.answer, evaluation: response.data}],
+		};
+	}
+
+	return undefined;
+};