From b09fe79cb71e7dd4373fd7813cd868b953e7cd2e Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Thu, 8 Feb 2024 11:43:01 +0000 Subject: [PATCH] Updated the InteractiveSpeaking to also work with the session persistence --- .../Exercises/InteractiveSpeaking.tsx | 168 +++++++++++++----- src/components/Exercises/index.tsx | 16 +- src/exams/Level.tsx | 2 +- src/exams/Listening.tsx | 2 +- src/exams/Reading.tsx | 2 +- src/exams/Speaking.tsx | 6 +- src/exams/Writing.tsx | 2 +- src/interfaces/exam.ts | 2 +- src/pages/(exam)/ExamPage.tsx | 8 +- src/pages/api/storage/insert.ts | 1 - src/pages/record.tsx | 5 +- src/utils/evaluation.ts | 20 ++- 12 files changed, 166 insertions(+), 68 deletions(-) diff --git a/src/components/Exercises/InteractiveSpeaking.tsx b/src/components/Exercises/InteractiveSpeaking.tsx index 3fe70479..a4c7ea22 100644 --- a/src/components/Exercises/InteractiveSpeaking.tsx +++ b/src/components/Exercises/InteractiveSpeaking.tsx @@ -5,6 +5,8 @@ import {BsCheckCircleFill, BsMicFill, BsPauseCircle, BsPlayCircle, BsTrashFill} import dynamic from "next/dynamic"; import Button from "../Low/Button"; import useExamStore from "@/stores/examStore"; +import {downloadBlob} from "@/utils/evaluation"; +import axios from "axios"; const Waveform = dynamic(() => import("../Waveform"), {ssr: false}); const ReactMediaRecorder = dynamic(() => import("react-media-recorder").then((mod) => mod.ReactMediaRecorder), { @@ -14,9 +16,11 @@ const ReactMediaRecorder = dynamic(() => import("react-media-recorder").then((mo export default function InteractiveSpeaking({ id, title, + examID, text, type, prompts, + userSolutions, updateIndex, onNext, onBack, @@ -24,21 +28,109 @@ export default function InteractiveSpeaking({ const [recordingDuration, setRecordingDuration] = useState(0); const [isRecording, setIsRecording] = useState(false); const [mediaBlob, setMediaBlob] = useState(); - const [answers, setAnswers] = useState<{prompt: string; blob: string}[]>([]); + const [answers, setAnswers] = useState<{prompt: string; blob: string; questionIndex: number}[]>([]); + const [isLoading, setIsLoading] = useState(false); const {questionIndex, setQuestionIndex} = useExamStore((state) => state); + const {userSolutions: storeUserSolutions, setUserSolutions} = useExamStore((state) => state); const hasExamEnded = useExamStore((state) => state.hasExamEnded); + const saveToStorage = async (previousURL?: string) => { + if (mediaBlob && mediaBlob.startsWith("blob")) { + const blobBuffer = await downloadBlob(mediaBlob); + const audioFile = new File([blobBuffer], "audio.wav", {type: "audio/wav"}); + + const seed = Math.random().toString().replace("0.", ""); + + const formData = new FormData(); + formData.append("audio", audioFile, `${seed}.wav`); + formData.append("root", "speaking_recordings"); + + const config = { + headers: { + "Content-Type": "audio/wav", + }, + }; + + const response = await axios.post<{path: string}>("/api/storage/insert", formData, config); + if (previousURL && !previousURL.startsWith("blob")) await axios.post("/api/storage/delete", {path: previousURL}); + return response.data.path; + } + + return undefined; + }; + + const back = async () => { + setIsLoading(true); + + const answer = await saveAnswer(questionIndex); + if (questionIndex - 1 >= 0) { + setQuestionIndex(questionIndex - 1); + setIsLoading(false); + + return; + } + setIsLoading(false); + + onBack({ + exercise: id, + solutions: [...answers.filter((x) => x.questionIndex !== questionIndex), answer], + score: {correct: 1, total: 1, missing: 0}, + type, + }); + }; + + const next = async () => { + setIsLoading(true); + + const answer = await saveAnswer(questionIndex); + if (questionIndex + 1 < prompts.length) { + setQuestionIndex(questionIndex + 1); + setIsLoading(false); + + return; + } + setIsLoading(false); + + onNext({ + exercise: id, + solutions: [...answers.filter((x) => x.questionIndex !== questionIndex), answer], + score: {correct: 1, total: 1, missing: 0}, + type, + }); + }; + + useEffect(() => { + if (userSolutions.length > 0 && answers.length === 0) { + console.log(userSolutions); + const solutions = userSolutions as unknown as typeof answers; + setAnswers(solutions); + + if (!mediaBlob) setMediaBlob(solutions.find((x) => x.questionIndex === questionIndex)?.blob); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [userSolutions, mediaBlob, answers]); + + useEffect(() => { + console.log({answers}); + }, [answers]); + useEffect(() => { if (updateIndex) updateIndex(questionIndex); }, [questionIndex, updateIndex]); useEffect(() => { if (hasExamEnded) { + const answer = { + questionIndex, + prompt: prompts[questionIndex].text, + blob: mediaBlob!, + }; + onNext({ exercise: id, - solutions: mediaBlob ? [{id, solution: mediaBlob}] : [], + solutions: [...answers.filter((x) => x.questionIndex !== questionIndex), answer], score: {correct: 1, total: 1, missing: 0}, type, }); @@ -60,19 +152,38 @@ export default function InteractiveSpeaking({ }, [isRecording]); useEffect(() => { - if (questionIndex === answers.length - 1) { - setMediaBlob(answers[questionIndex].blob); + if (questionIndex <= answers.length - 1) { + const blob = answers.find((x) => x.questionIndex === questionIndex)?.blob; + setMediaBlob(blob); } }, [answers, questionIndex]); - const saveAnswer = () => { + const saveAnswer = async (index: number) => { + const previousURL = answers.find((x) => x.questionIndex === questionIndex)?.blob; + const audioPath = await saveToStorage(previousURL); + const answer = { + questionIndex, prompt: prompts[questionIndex].text, - blob: mediaBlob!, + blob: audioPath ? audioPath : mediaBlob!, }; - setAnswers((prev) => [...prev, answer]); + setAnswers((prev) => [...prev.filter((x) => x.questionIndex !== index), answer]); setMediaBlob(undefined); + + setUserSolutions([ + ...storeUserSolutions.filter((x) => x.exercise !== id), + { + exercise: id, + solutions: [...answers.filter((x) => x.questionIndex !== questionIndex), answer], + score: {correct: 1, total: 1, missing: 0}, + module: "speaking", + exam: examID, + type, + }, + ]); + + return answer; }; return ( @@ -98,7 +209,7 @@ export default function InteractiveSpeaking({

Record your answer:

- {status === "idle" && ( + {status === "idle" && !mediaBlob && ( <>
{status === "idle" && ( @@ -177,9 +288,9 @@ export default function InteractiveSpeaking({
)} - {status === "stopped" && mediaBlobUrl && ( + {((status === "stopped" && mediaBlobUrl) || (status === "idle" && mediaBlob)) && ( <> - +
- -
diff --git a/src/components/Exercises/index.tsx b/src/components/Exercises/index.tsx index fd63f1f9..2b1a623b 100644 --- a/src/components/Exercises/index.tsx +++ b/src/components/Exercises/index.tsx @@ -22,6 +22,7 @@ import InteractiveSpeaking from "./InteractiveSpeaking"; const MatchSentences = dynamic(() => import("@/components/Exercises/MatchSentences"), {ssr: false}); export interface CommonProps { + examID?: string; updateIndex?: (internalIndex: number) => void; onNext: (userSolutions: UserSolution) => void; onBack: (userSolutions: UserSolution) => void; @@ -29,17 +30,18 @@ export interface CommonProps { export const renderExercise = ( exercise: Exercise, + examID: string, onNext: (userSolutions: UserSolution) => void, onBack: (userSolutions: UserSolution) => void, updateIndex?: (internalIndex: number) => void, ) => { switch (exercise.type) { case "fillBlanks": - return ; + return ; case "trueFalse": - return ; + return ; case "matchSentences": - return ; + return ; case "multipleChoice": return ( ); case "writeBlanks": - return ; + return ; case "writing": - return ; + return ; case "speaking": - return ; + return ; case "interactiveSpeaking": return ( -1 && exerciseIndex < exam.exercises.length && !showSolutions && - renderExercise(getExercise(), nextExercise, previousExercise, setCurrentQuestionIndex)} + renderExercise(getExercise(), exam.id, nextExercise, previousExercise, setCurrentQuestionIndex)} {exerciseIndex > -1 && exerciseIndex < exam.exercises.length && showSolutions && diff --git a/src/exams/Listening.tsx b/src/exams/Listening.tsx index 3189b402..ab8dfa0d 100644 --- a/src/exams/Listening.tsx +++ b/src/exams/Listening.tsx @@ -183,7 +183,7 @@ export default function Listening({exam, showSolutions = false, onFinish}: Props partIndex > -1 && exerciseIndex < exam.parts[partIndex].exercises.length && !showSolutions && - renderExercise(getExercise(), nextExercise, previousExercise, setCurrentQuestionIndex)} + renderExercise(getExercise(), exam.id, nextExercise, previousExercise, setCurrentQuestionIndex)} {/* Solution renderer */} {exerciseIndex > -1 && diff --git a/src/exams/Reading.tsx b/src/exams/Reading.tsx index 16a6d457..58d3f728 100644 --- a/src/exams/Reading.tsx +++ b/src/exams/Reading.tsx @@ -243,7 +243,7 @@ export default function Reading({exam, showSolutions = false, onFinish}: Props) partIndex > -1 && exerciseIndex < exam.parts[partIndex].exercises.length && !showSolutions && - renderExercise(getExercise(), nextExercise, previousExercise, setCurrentQuestionIndex)} + renderExercise(getExercise(), exam.id, nextExercise, previousExercise, setCurrentQuestionIndex)} {exerciseIndex > -1 && partIndex > -1 && diff --git a/src/exams/Speaking.tsx b/src/exams/Speaking.tsx index 129892b6..d69c707f 100644 --- a/src/exams/Speaking.tsx +++ b/src/exams/Speaking.tsx @@ -80,18 +80,18 @@ export default function Speaking({exam, showSolutions = false, onFinish}: Props) return ( <>
- {/* */} + /> {exerciseIndex > -1 && exerciseIndex < exam.exercises.length && !showSolutions && - renderExercise(getExercise(), nextExercise, previousExercise, setCurrentQuestionIndex)} + renderExercise(getExercise(), exam.id, nextExercise, previousExercise, setCurrentQuestionIndex)} {exerciseIndex > -1 && exerciseIndex < exam.exercises.length && showSolutions && diff --git a/src/exams/Writing.tsx b/src/exams/Writing.tsx index d9ec398f..2efa9244 100644 --- a/src/exams/Writing.tsx +++ b/src/exams/Writing.tsx @@ -81,7 +81,7 @@ export default function Writing({exam, showSolutions = false, onFinish}: Props) {exerciseIndex > -1 && exerciseIndex < exam.exercises.length && !showSolutions && - renderExercise(getExercise(), nextExercise, previousExercise)} + renderExercise(getExercise(), exam.id, nextExercise, previousExercise)} {exerciseIndex > -1 && exerciseIndex < exam.exercises.length && showSolutions && diff --git a/src/interfaces/exam.ts b/src/interfaces/exam.ts index 2f3e2128..bafb809f 100644 --- a/src/interfaces/exam.ts +++ b/src/interfaces/exam.ts @@ -164,7 +164,7 @@ export interface InteractiveSpeakingExercise { prompts: {text: string; video_url: string}[]; userSolutions: { id: string; - solution: {question: string; answer: string}[]; + solution: {questionIndex: number; question: string; answer: string}[]; evaluation?: InteractiveSpeakingEvaluation; }[]; } diff --git a/src/pages/(exam)/ExamPage.tsx b/src/pages/(exam)/ExamPage.tsx index f51ecd68..7ac609c1 100644 --- a/src/pages/(exam)/ExamPage.tsx +++ b/src/pages/(exam)/ExamPage.tsx @@ -181,8 +181,8 @@ export default function ExamPage({page}: Props) { id: solution.id || uuidv4(), timeSpent, session: sessionId, - exam: solution.exam!, - module: solution.module!, + exam: exam!.id, + module: exam!.module, user: user?.id || "", date: new Date().getTime(), ...(assignment ? {assignment: assignment.id} : {}), @@ -328,6 +328,8 @@ export default function ExamPage({page}: Props) { }; answers.forEach((x) => { + console.log({x}); + scores[x.module!] = { total: scores[x.module!].total + x.score.total, correct: scores[x.module!].correct + x.score.correct, @@ -366,6 +368,8 @@ export default function ExamPage({page}: Props) { onViewResults={() => { setShowSolutions(true); setModuleIndex(0); + setExerciseIndex(["reading", "listening"].includes(exams[0].module) ? -1 : 0); + setPartIndex(exams[0].module === "listening" ? -1 : 0); setExam(exams[0]); }} scores={aggregateScoresByModule(userSolutions)} diff --git a/src/pages/api/storage/insert.ts b/src/pages/api/storage/insert.ts index 7baa8196..bd664ff8 100644 --- a/src/pages/api/storage/insert.ts +++ b/src/pages/api/storage/insert.ts @@ -28,7 +28,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { } const audioFile = files.audio; - console.log({fields}, (audioFile as any).path); const audioFileRef = ref(storage, `${fields.root}/${(audioFile as any).path.split("/").pop()!.replace("upload_", "")}`); const binary = fs.readFileSync((audioFile as any).path).buffer; diff --git a/src/pages/record.tsx b/src/pages/record.tsx index 31966d57..d5c5d97f 100644 --- a/src/pages/record.tsx +++ b/src/pages/record.tsx @@ -178,7 +178,10 @@ export default function History({user}: {user: User}) { const {timeSpent, session} = dateStats[0]; const selectExam = () => { - const examPromises = uniqBy(dateStats, "exam").map((stat) => getExamById(stat.module, stat.exam)); + const examPromises = uniqBy(dateStats, "exam").map((stat) => { + console.log({stat}); + return getExamById(stat.module, stat.exam); + }); Promise.all(examPromises).then((exams) => { if (exams.every((x) => !!x)) { diff --git a/src/utils/evaluation.ts b/src/utils/evaluation.ts index 932d4e0d..453236c6 100644 --- a/src/utils/evaluation.ts +++ b/src/utils/evaluation.ts @@ -53,9 +53,12 @@ export const downloadBlob = async (url: string): Promise => { const evaluateSpeakingExercise = async (exercise: SpeakingExercise, exerciseId: string, solution: UserSolution, id: string) => { const formData = new FormData(); - const audioBlob = await downloadBlob(solution.solutions[0].solution.trim()); + const url = solution.solutions[0].solution.trim() as string; + const audioBlob = await downloadBlob(url); const audioFile = new File([audioBlob], "audio.wav", {type: "audio/wav"}); + if (url && !url.startsWith("blob")) await axios.post("/api/storage/delete", {path: url}); + formData.append("audio", audioFile, "audio.wav"); const evaluationQuestion = @@ -87,10 +90,15 @@ const evaluateSpeakingExercise = async (exercise: SpeakingExercise, exerciseId: }; const evaluateInteractiveSpeakingExercise = async (exerciseId: string, solution: UserSolution, id: string) => { - const promiseParts = solution.solutions.map(async (x: {prompt: string; blob: string}) => ({ - question: x.prompt, - answer: await downloadBlob(x.blob), - })); + const promiseParts = solution.solutions.map(async (x: {prompt: string; blob: string}) => { + const blob = await downloadBlob(x.blob); + if (!x.blob.startsWith("blob")) await axios.post("/api/storage/delete", {path: x.blob}); + + return { + question: x.prompt, + answer: blob, + }; + }); const body = await Promise.all(promiseParts); const formData = new FormData(); @@ -111,6 +119,7 @@ const evaluateInteractiveSpeakingExercise = async (exerciseId: string, solution: }; const response = await axios.post("/api/evaluate/interactiveSpeaking", formData, config); + console.log({data: response.data, status: response.status}); if (response.status === 200) { return { @@ -120,6 +129,7 @@ const evaluateInteractiveSpeakingExercise = async (exerciseId: string, solution: missing: 0, total: 100, }, + module: "speaking", solutions: [{id: exerciseId, solution: response.data ? response.data.answer : null, evaluation: response.data}], }; }