diff --git a/src/components/Exercises/FillBlanks.tsx b/src/components/Exercises/FillBlanks.tsx index 1f529140..f9e14a2a 100644 --- a/src/components/Exercises/FillBlanks.tsx +++ b/src/components/Exercises/FillBlanks.tsx @@ -130,6 +130,7 @@ export default function FillBlanks({
{(!!currentBlankId || isDrawerShowing) && ( ({word, isDisabled: allowRepetition ? false : answers.map((x) => x.solution).includes(word)}))} previouslySelectedWord={currentBlankId ? answers.find((x) => x.id === currentBlankId)?.solution : undefined} diff --git a/src/components/Exercises/InteractiveSpeaking.tsx b/src/components/Exercises/InteractiveSpeaking.tsx new file mode 100644 index 00000000..b7fb2ed0 --- /dev/null +++ b/src/components/Exercises/InteractiveSpeaking.tsx @@ -0,0 +1,240 @@ +import {InteractiveSpeakingExercise} from "@/interfaces/exam"; +import {CommonProps} from "."; +import {useEffect, useState} from "react"; +import {BsCheckCircleFill, BsMicFill, BsPauseCircle, BsPlayCircle, BsTrashFill} from "react-icons/bs"; +import dynamic from "next/dynamic"; +import Button from "../Low/Button"; +import useExamStore from "@/stores/examStore"; + +const Waveform = dynamic(() => import("../Waveform"), {ssr: false}); +const ReactMediaRecorder = dynamic(() => import("react-media-recorder").then((mod) => mod.ReactMediaRecorder), { + ssr: false, +}); + +export default function InteractiveSpeaking({id, title, text, type, prompts, onNext, onBack}: InteractiveSpeakingExercise & CommonProps) { + const [recordingDuration, setRecordingDuration] = useState(0); + const [isRecording, setIsRecording] = useState(false); + const [mediaBlob, setMediaBlob] = useState(); + const [promptIndex, setPromptIndex] = useState(0); + const [answers, setAnswers] = useState<{prompt: string; blob: string}[]>([]); + + const hasExamEnded = useExamStore((state) => state.hasExamEnded); + + useEffect(() => { + if (hasExamEnded) { + onNext({ + exercise: id, + solutions: mediaBlob ? [{id, solution: mediaBlob}] : [], + score: {correct: 1, total: 1, missing: 0}, + type, + }); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [hasExamEnded]); + + useEffect(() => { + let recordingInterval: NodeJS.Timer | undefined = undefined; + if (isRecording) { + recordingInterval = setInterval(() => setRecordingDuration((prev) => prev + 1), 1000); + } else if (recordingInterval) { + clearInterval(recordingInterval); + } + + return () => { + if (recordingInterval) clearInterval(recordingInterval); + }; + }, [isRecording]); + + useEffect(() => { + if (promptIndex === answers.length - 1) { + setMediaBlob(answers[promptIndex].blob); + } + }, [answers, promptIndex]); + + const saveAnswer = () => { + const answer = { + prompt: prompts[promptIndex].text, + blob: mediaBlob!, + }; + + setAnswers((prev) => [...prev, answer]); + setMediaBlob(undefined); + }; + + return ( +
+
+
+ {title} +
+ {prompts && prompts.length > 0 && ( +
+ +
+ )} +
+ + setMediaBlob(blob)} + render={({status, startRecording, stopRecording, pauseRecording, resumeRecording, clearBlobUrl, mediaBlobUrl}) => ( +
+

Record your answer:

+
+ {status === "idle" && ( + <> +
+ {status === "idle" && ( + { + setRecordingDuration(0); + startRecording(); + setIsRecording(true); + }} + className="h-5 w-5 text-mti-gray-cool cursor-pointer" + /> + )} + + )} + {status === "recording" && ( + <> +
+ + {Math.floor(recordingDuration / 60) + .toString(10) + .padStart(2, "0")} + : + {Math.floor(recordingDuration % 60) + .toString(10) + .padStart(2, "0")} + +
+
+
+ { + setIsRecording(false); + pauseRecording(); + }} + className="text-red-500 w-8 h-8 cursor-pointer" + /> + { + setIsRecording(false); + stopRecording(); + }} + className="text-mti-purple-light w-8 h-8 cursor-pointer" + /> +
+ + )} + {status === "paused" && ( + <> +
+ + {Math.floor(recordingDuration / 60) + .toString(10) + .padStart(2, "0")} + : + {Math.floor(recordingDuration % 60) + .toString(10) + .padStart(2, "0")} + +
+
+
+ { + setIsRecording(true); + resumeRecording(); + }} + className="text-mti-purple-light w-8 h-8 cursor-pointer" + /> + { + setIsRecording(false); + stopRecording(); + }} + className="text-mti-purple-light w-8 h-8 cursor-pointer" + /> +
+ + )} + {status === "stopped" && mediaBlobUrl && ( + <> + +
+ { + setRecordingDuration(0); + clearBlobUrl(); + setMediaBlob(undefined); + }} + /> + + { + clearBlobUrl(); + setRecordingDuration(0); + startRecording(); + setIsRecording(true); + setMediaBlob(undefined); + }} + className="h-5 w-5 text-mti-gray-cool cursor-pointer" + /> +
+ + )} +
+
+ )} + /> + +
+ + +
+
+ ); +} diff --git a/src/components/Exercises/index.tsx b/src/components/Exercises/index.tsx index 9eb0b1df..89cc70af 100644 --- a/src/components/Exercises/index.tsx +++ b/src/components/Exercises/index.tsx @@ -1,6 +1,7 @@ import { Exercise, FillBlanksExercise, + InteractiveSpeakingExercise, MatchSentencesExercise, MultipleChoiceExercise, SpeakingExercise, @@ -16,6 +17,7 @@ import WriteBlanks from "./WriteBlanks"; import Writing from "./Writing"; import Speaking from "./Speaking"; import TrueFalse from "./TrueFalse"; +import InteractiveSpeaking from "./InteractiveSpeaking"; const MatchSentences = dynamic(() => import("@/components/Exercises/MatchSentences"), {ssr: false}); @@ -40,5 +42,7 @@ export const renderExercise = (exercise: Exercise, onNext: (userSolutions: UserS return ; case "speaking": return ; + case "interactiveSpeaking": + return ; } }; diff --git a/src/components/Solutions/InteractiveSpeaking.tsx b/src/components/Solutions/InteractiveSpeaking.tsx new file mode 100644 index 00000000..4113260a --- /dev/null +++ b/src/components/Solutions/InteractiveSpeaking.tsx @@ -0,0 +1,124 @@ +/* eslint-disable @next/next/no-img-element */ +import {InteractiveSpeakingExercise} from "@/interfaces/exam"; +import {CommonProps} from "."; +import {useEffect, useState} from "react"; +import Button from "../Low/Button"; +import dynamic from "next/dynamic"; +import axios from "axios"; +import {speakingReverseMarking} from "@/utils/score"; + +const Waveform = dynamic(() => import("../Waveform"), {ssr: false}); + +export default function InteractiveSpeaking({ + id, + type, + title, + text, + prompts, + userSolutions, + onNext, + onBack, +}: InteractiveSpeakingExercise & CommonProps) { + const [solutionsURL, setSolutionsURL] = useState([]); + + useEffect(() => { + if (userSolutions && userSolutions.length > 0) { + Promise.all(userSolutions[0].solution.map((x) => axios.post(`/api/speaking`, {path: x.answer}, {responseType: "arraybuffer"}))).then( + (values) => { + setSolutionsURL( + values.map(({data}) => { + const blob = new Blob([data], {type: "audio/wav"}); + const url = URL.createObjectURL(blob); + + return url; + }), + ); + }, + ); + } + }, [userSolutions]); + + return ( + <> +
+
+
+ {title} +
+
+ You should talk about the following things: +
+ {prompts.map((x, index) => ( +
+ + {x.text} +
+ ))} +
+
+
+ +
+
+ {solutionsURL.map((x, index) => ( +
+
+ +
+
+ ))} +
+ + {userSolutions && userSolutions.length > 0 && ( +
+
+ {Object.keys(userSolutions[0].evaluation!.task_response).map((key) => ( +
+ {key}: Level {userSolutions[0].evaluation!.task_response[key]} +
+ ))} +
+
+ {userSolutions[0].evaluation!.comment} +
+
+ )} +
+
+ +
+ + +
+ + ); +} diff --git a/src/components/Solutions/index.tsx b/src/components/Solutions/index.tsx index 225b3ace..5c3db040 100644 --- a/src/components/Solutions/index.tsx +++ b/src/components/Solutions/index.tsx @@ -1,6 +1,7 @@ import { Exercise, FillBlanksExercise, + InteractiveSpeakingExercise, MatchSentencesExercise, MultipleChoiceExercise, SpeakingExercise, @@ -11,6 +12,7 @@ import { } from "@/interfaces/exam"; import dynamic from "next/dynamic"; import FillBlanks from "./FillBlanks"; +import InteractiveSpeaking from "./InteractiveSpeaking"; import MultipleChoice from "./MultipleChoice"; import Speaking from "./Speaking"; import TrueFalseSolution from "./TrueFalse"; @@ -25,6 +27,8 @@ export interface CommonProps { } export const renderSolution = (exercise: Exercise, onNext: () => void, onBack: () => void) => { + console.log(exercise); + switch (exercise.type) { case "fillBlanks": return ; @@ -40,5 +44,7 @@ export const renderSolution = (exercise: Exercise, onNext: () => void, onBack: ( return ; case "speaking": return ; + case "interactiveSpeaking": + return ; } }; diff --git a/src/interfaces/exam.ts b/src/interfaces/exam.ts index 76bcad67..5a277d43 100644 --- a/src/interfaces/exam.ts +++ b/src/interfaces/exam.ts @@ -68,7 +68,8 @@ export type Exercise = | MultipleChoiceExercise | WriteBlanksExercise | WritingExercise - | SpeakingExercise; + | SpeakingExercise + | InteractiveSpeakingExercise; export interface Evaluation { comment: string; @@ -108,13 +109,13 @@ export interface SpeakingExercise { export interface InteractiveSpeakingExercise { id: string; - type: "speaking"; + type: "interactiveSpeaking"; title: string; text: string; prompts: {text: string; video_url: string}[]; userSolutions: { id: string; - solution: string; + solution: {question: string; answer: string}[]; evaluation?: Evaluation; }[]; } diff --git a/src/pages/api/evaluate/interactiveSpeaking.ts b/src/pages/api/evaluate/interactiveSpeaking.ts new file mode 100644 index 00000000..e1db5851 --- /dev/null +++ b/src/pages/api/evaluate/interactiveSpeaking.ts @@ -0,0 +1,59 @@ +// Next.js API route support: https://nextjs.org/docs/api-routes/introduction +import type {NextApiRequest, NextApiResponse} from "next"; +import {withIronSessionApiRoute} from "iron-session/next"; +import {sessionOptions} from "@/lib/session"; +import axios from "axios"; +import formidable from "formidable-serverless"; +import {getStorage, ref, uploadBytes} from "firebase/storage"; +import fs from "fs"; +import {app} from "@/firebase"; + +export default withIronSessionApiRoute(handler, sessionOptions); + +async function handler(req: NextApiRequest, res: NextApiResponse) { + if (!req.session.user) { + res.status(401).json({ok: false}); + return; + } + + const storage = getStorage(app); + + const form = formidable({keepExtensions: true}); + await form.parse(req, async (err: any, fields: any, files: any) => { + if (err) console.log(err); + + const uploadingAudios = await Promise.all( + Object.keys(files).map(async (fileID: string) => { + const audioFile = files[fileID]; + const questionID = fileID.replace("answer_", "question_"); + + const audioFileRef = ref(storage, `speaking_recordings/${(audioFile as any).path.replace("upload_", "")}`); + + const binary = fs.readFileSync((audioFile as any).path).buffer; + const snapshot = await uploadBytes(audioFileRef, binary); + + fs.rmSync((audioFile as any).path); + + return {question: fields[questionID], answer: snapshot.metadata.fullPath}; + }), + ); + + const backendRequest = await axios.post( + `${process.env.BACKEND_URL}/speaking_task_3`, + {answers: uploadingAudios}, + { + headers: { + Authorization: `Bearer ${process.env.BACKEND_JWT}`, + }, + }, + ); + + res.status(200).json({...backendRequest.data, answer: uploadingAudios}); + }); +} + +export const config = { + api: { + bodyParser: false, + }, +}; diff --git a/src/pages/exercises.tsx b/src/pages/exercises.tsx index 4543aaac..2265aace 100644 --- a/src/pages/exercises.tsx +++ b/src/pages/exercises.tsx @@ -71,7 +71,7 @@ export default function Page() { const {user} = useUser({redirectTo: "/login"}); - useEffect(() => console.log({examId: exam?.id}), [exam]); + useEffect(() => console.log({examId: exam?.id, exam}), [exam]); useEffect(() => setSessionId(uuidv4()), []); useEffect(() => { diff --git a/src/utils/evaluation.ts b/src/utils/evaluation.ts index b29a10c4..06421b08 100644 --- a/src/utils/evaluation.ts +++ b/src/utils/evaluation.ts @@ -1,4 +1,4 @@ -import {Evaluation, Exam, SpeakingExercise, UserSolution, WritingExercise} from "@/interfaces/exam"; +import {Evaluation, Exam, InteractiveSpeakingExercise, SpeakingExercise, UserSolution, WritingExercise} from "@/interfaces/exam"; import axios from "axios"; import {speakingReverseMarking, writingReverseMarking} from "./score"; @@ -30,16 +30,23 @@ export const evaluateSpeakingAnswer = async (exams: Exam[], examId: string, exer const speakingExam = exams.find((x) => x.id === examId)!; const exercise = speakingExam.exercises.find((x) => x.id === exerciseId); - if (exercise?.type === "speaking") { - return await evaluateSpeakingExercise(exercise, exerciseId, solution); + switch (exercise?.type) { + case "speaking": + return await evaluateSpeakingExercise(exercise, exerciseId, solution); + case "interactiveSpeaking": + return await evaluateInteractiveSpeakingExercise(exerciseId, solution); + default: + return undefined; } +}; - return undefined; +const downloadBlob = async (url: string): Promise => { + const blobResponse = await axios.get(url, {responseType: "arraybuffer"}); + return Buffer.from(blobResponse.data, "binary"); }; const evaluateSpeakingExercise = async (exercise: SpeakingExercise, exerciseId: string, solution: UserSolution) => { - const blobResponse = await axios.get(solution.solutions[0].solution.trim(), {responseType: "arraybuffer"}); - const audioBlob = Buffer.from(blobResponse.data, "binary"); + const audioBlob = await downloadBlob(solution.solutions[0].solution.trim()); const audioFile = new File([audioBlob], "audio.wav", {type: "audio/wav"}); const formData = new FormData(); @@ -68,3 +75,43 @@ const evaluateSpeakingExercise = async (exercise: SpeakingExercise, exerciseId: return undefined; }; + +const evaluateInteractiveSpeakingExercise = async (exerciseId: string, solution: UserSolution) => { + const promiseParts = solution.solutions.map(async (x: {prompt: string; blob: string}) => ({ + question: x.prompt, + answer: await downloadBlob(x.blob), + })); + const body = await Promise.all(promiseParts); + + const formData = new FormData(); + body.forEach(({question, answer}) => { + const seed = Math.random().toString().replace("0.", ""); + + const audioFile = new File([answer], `${seed}.wav`, {type: "audio/wav"}); + + formData.append(`question_${seed}`, question); + formData.append(`answer_${seed}`, audioFile, `${seed}.wav`); + }); + + const config = { + headers: { + "Content-Type": "audio/mp3", + }, + }; + + const response = await axios.post("/api/evaluate/interactiveSpeaking", formData, config); + + if (response.status === 200) { + return { + ...solution, + score: { + correct: speakingReverseMarking[response.data.overall] || 0, + missing: 0, + total: 100, + }, + solutions: [{id: exerciseId, solution: response.data.answer, evaluation: response.data}], + }; + } + + return undefined; +};