From 2a9e2040412f585030b663be54780d597be57a98 Mon Sep 17 00:00:00 2001 From: Tiago Ribeiro Date: Wed, 7 Feb 2024 17:15:41 +0000 Subject: [PATCH] Updated the Speaking to also work the with exam session persistence --- .../Exercises/InteractiveSpeaking.tsx | 29 ++--- src/components/Exercises/Speaking.tsx | 106 ++++++++++++------ src/exams/Speaking.tsx | 4 +- src/pages/(exam)/ExamPage.tsx | 14 ++- src/pages/api/sessions/[id].ts | 1 + src/pages/api/storage/delete.ts | 28 +++++ src/pages/api/storage/insert.ts | 46 ++++++++ src/utils/evaluation.ts | 7 +- 8 files changed, 177 insertions(+), 58 deletions(-) create mode 100644 src/pages/api/storage/delete.ts create mode 100644 src/pages/api/storage/insert.ts diff --git a/src/components/Exercises/InteractiveSpeaking.tsx b/src/components/Exercises/InteractiveSpeaking.tsx index c9f324fe..3fe70479 100644 --- a/src/components/Exercises/InteractiveSpeaking.tsx +++ b/src/components/Exercises/InteractiveSpeaking.tsx @@ -24,14 +24,15 @@ export default function InteractiveSpeaking({ const [recordingDuration, setRecordingDuration] = useState(0); const [isRecording, setIsRecording] = useState(false); const [mediaBlob, setMediaBlob] = useState(); - const [promptIndex, setPromptIndex] = useState(0); const [answers, setAnswers] = useState<{prompt: string; blob: string}[]>([]); + const {questionIndex, setQuestionIndex} = useExamStore((state) => state); + const hasExamEnded = useExamStore((state) => state.hasExamEnded); useEffect(() => { - if (updateIndex) updateIndex(promptIndex); - }, [promptIndex, updateIndex]); + if (updateIndex) updateIndex(questionIndex); + }, [questionIndex, updateIndex]); useEffect(() => { if (hasExamEnded) { @@ -59,14 +60,14 @@ export default function InteractiveSpeaking({ }, [isRecording]); useEffect(() => { - if (promptIndex === answers.length - 1) { - setMediaBlob(answers[promptIndex].blob); + if (questionIndex === answers.length - 1) { + setMediaBlob(answers[questionIndex].blob); } - }, [answers, promptIndex]); + }, [answers, questionIndex]); const saveAnswer = () => { const answer = { - prompt: prompts[promptIndex].text, + prompt: prompts[questionIndex].text, blob: mediaBlob!, }; @@ -82,8 +83,8 @@ export default function InteractiveSpeaking({ {prompts && prompts.length > 0 && (
-
)} @@ -91,7 +92,7 @@ export default function InteractiveSpeaking({ setMediaBlob(blob)} render={({status, startRecording, stopRecording, pauseRecording, resumeRecording, clearBlobUrl, mediaBlobUrl}) => (
@@ -227,8 +228,8 @@ export default function InteractiveSpeaking({ disabled={!mediaBlob} onClick={() => { saveAnswer(); - if (promptIndex + 1 < prompts.length) { - setPromptIndex((prev) => prev + 1); + if (questionIndex + 1 < prompts.length) { + setQuestionIndex(questionIndex + 1); return; } onNext({ @@ -236,7 +237,7 @@ export default function InteractiveSpeaking({ solutions: [ ...answers, { - prompt: prompts[promptIndex].text, + prompt: prompts[questionIndex].text, blob: mediaBlob!, }, ], @@ -245,7 +246,7 @@ export default function InteractiveSpeaking({ }); }} className="max-w-[200px] self-end w-full"> - {promptIndex + 1 < prompts.length ? "Next Prompt" : "Submit"} + {questionIndex + 1 < prompts.length ? "Next Prompt" : "Submit"}
diff --git a/src/components/Exercises/Speaking.tsx b/src/components/Exercises/Speaking.tsx index 03c79fd0..40cb55a9 100644 --- a/src/components/Exercises/Speaking.tsx +++ b/src/components/Exercises/Speaking.tsx @@ -5,28 +5,58 @@ import {BsCheckCircleFill, BsMicFill, BsPauseCircle, BsPlayCircle, BsTrashFill} import dynamic from "next/dynamic"; import Button from "../Low/Button"; import useExamStore from "@/stores/examStore"; +import {downloadBlob} from "@/utils/evaluation"; +import axios from "axios"; const Waveform = dynamic(() => import("../Waveform"), {ssr: false}); const ReactMediaRecorder = dynamic(() => import("react-media-recorder").then((mod) => mod.ReactMediaRecorder), { ssr: false, }); -export default function Speaking({id, title, text, video_url, type, prompts, onNext, onBack}: SpeakingExercise & CommonProps) { +export default function Speaking({id, title, text, video_url, type, prompts, userSolutions, onNext, onBack}: SpeakingExercise & CommonProps) { const [recordingDuration, setRecordingDuration] = useState(0); const [isRecording, setIsRecording] = useState(false); const [mediaBlob, setMediaBlob] = useState(); + const [audioURL, setAudioURL] = useState(); + const [isLoading, setIsLoading] = useState(false); const hasExamEnded = useExamStore((state) => state.hasExamEnded); - useEffect(() => { - if (hasExamEnded) { - onNext({ - exercise: id, - solutions: mediaBlob ? [{id, solution: mediaBlob}] : [], - score: {correct: 1, total: 1, missing: 0}, - type, - }); + const saveToStorage = async () => { + if (mediaBlob && mediaBlob.startsWith("blob")) { + const blobBuffer = await downloadBlob(mediaBlob); + const audioFile = new File([blobBuffer], "audio.wav", {type: "audio/wav"}); + + const seed = Math.random().toString().replace("0.", ""); + + const formData = new FormData(); + formData.append("audio", audioFile, `${seed}.wav`); + formData.append("root", "speaking_recordings"); + + const config = { + headers: { + "Content-Type": "audio/wav", + }, + }; + + const response = await axios.post<{path: string}>("/api/storage/insert", formData, config); + if (audioURL) await axios.post("/api/storage/delete", {path: audioURL}); + return response.data.path; } + + return undefined; + }; + + useEffect(() => { + if (userSolutions.length > 0) { + const {solution} = userSolutions[0] as {solution?: string}; + if (solution && !mediaBlob) setMediaBlob(solution); + if (solution && !solution.startsWith("blob")) setAudioURL(solution); + } + }, [userSolutions, mediaBlob]); + + useEffect(() => { + if (hasExamEnded) next(); // eslint-disable-next-line react-hooks/exhaustive-deps }, [hasExamEnded]); @@ -43,6 +73,32 @@ export default function Speaking({id, title, text, video_url, type, prompts, onN }; }, [isRecording]); + const next = async () => { + setIsLoading(true); + const storagePath = await saveToStorage(); + setIsLoading(false); + + onNext({ + exercise: id, + solutions: storagePath ? [{id, solution: storagePath}] : [], + score: {correct: 1, total: 1, missing: 0}, + type, + }); + }; + + const back = async () => { + setIsLoading(true); + const storagePath = await saveToStorage(); + setIsLoading(false); + + onBack({ + exercise: id, + solutions: storagePath ? [{id, solution: storagePath}] : [], + score: {correct: 1, total: 1, missing: 0}, + type, + }); + }; + return (
@@ -89,7 +145,7 @@ export default function Speaking({id, title, text, video_url, type, prompts, onN

Record your answer:

- {status === "idle" && ( + {status === "idle" && !mediaBlob && ( <>
{status === "idle" && ( @@ -168,9 +224,9 @@ export default function Speaking({id, title, text, video_url, type, prompts, onN
)} - {status === "stopped" && mediaBlobUrl && ( + {((status === "stopped" && mediaBlobUrl) || (status === "idle" && mediaBlob)) && ( <> - +
- -
diff --git a/src/exams/Speaking.tsx b/src/exams/Speaking.tsx index ecd52000..129892b6 100644 --- a/src/exams/Speaking.tsx +++ b/src/exams/Speaking.tsx @@ -80,14 +80,14 @@ export default function Speaking({exam, showSolutions = false, onFinish}: Props) return ( <>
- + /> */} {exerciseIndex > -1 && exerciseIndex < exam.exercises.length && !showSolutions && diff --git a/src/pages/(exam)/ExamPage.tsx b/src/pages/(exam)/ExamPage.tsx index fd411913..f51ecd68 100644 --- a/src/pages/(exam)/ExamPage.tsx +++ b/src/pages/(exam)/ExamPage.tsx @@ -98,13 +98,22 @@ export default function ExamPage({page}: Props) { }, [exams, setUserSolutions, userSolutions]); useEffect(() => { - if (sessionId.length > 0 && userSolutions.length > 0 && selectedModules.length > 0 && exams.length > 0 && !!exam && timeSpent > 0) + if ( + sessionId.length > 0 && + userSolutions.length > 0 && + selectedModules.length > 0 && + exams.length > 0 && + !!exam && + timeSpent > 0 && + !showSolutions && + moduleIndex < selectedModules.length + ) saveSession(); // eslint-disable-next-line react-hooks/exhaustive-deps }, [assignment, exam, exams, moduleIndex, selectedModules, sessionId, userSolutions, user, exerciseIndex, partIndex, questionIndex]); useEffect(() => { - if (timeSpent % 20 === 0 && timeSpent > 0) saveSession(); + if (timeSpent % 20 === 0 && timeSpent > 0 && moduleIndex < selectedModules.length && !showSolutions) saveSession(); // eslint-disable-next-line react-hooks/exhaustive-deps }, [timeSpent]); @@ -282,7 +291,6 @@ export default function ExamPage({page}: Props) { setUserSolutions([...userSolutions.filter((x) => !solutionIds.includes(x.exercise)), ...solutions]); setModuleIndex(moduleIndex + 1); - // TODO: Solve the issue for the listening where it should start with -1 setPartIndex(-1); setExerciseIndex(-1); setQuestionIndex(0); diff --git a/src/pages/api/sessions/[id].ts b/src/pages/api/sessions/[id].ts index 58e1af15..c564e84f 100644 --- a/src/pages/api/sessions/[id].ts +++ b/src/pages/api/sessions/[id].ts @@ -4,6 +4,7 @@ import {app} from "@/firebase"; import {getFirestore, doc, getDoc, deleteDoc} from "firebase/firestore"; import {withIronSessionApiRoute} from "iron-session/next"; import {sessionOptions} from "@/lib/session"; +import {Session} from "@/hooks/useSessions"; const db = getFirestore(app); diff --git a/src/pages/api/storage/delete.ts b/src/pages/api/storage/delete.ts new file mode 100644 index 00000000..b22bf231 --- /dev/null +++ b/src/pages/api/storage/delete.ts @@ -0,0 +1,28 @@ +// Next.js API route support: https://nextjs.org/docs/api-routes/introduction +import type {NextApiRequest, NextApiResponse} from "next"; +import {app} from "@/firebase"; +import {getFirestore, doc, getDoc, deleteDoc} from "firebase/firestore"; +import {withIronSessionApiRoute} from "iron-session/next"; +import {sessionOptions} from "@/lib/session"; +import {Session} from "@/hooks/useSessions"; +import {deleteObject, getStorage, ref} from "firebase/storage"; + +const storage = getStorage(app); + +export default withIronSessionApiRoute(handler, sessionOptions); + +async function handler(req: NextApiRequest, res: NextApiResponse) { + if (req.method === "POST") return post(req, res); +} + +async function post(req: NextApiRequest, res: NextApiResponse) { + if (!req.session.user) { + res.status(401).json({ok: false}); + return; + } + + const {path} = req.body as {path: string}; + await deleteObject(ref(storage, path)); + + return res.status(200).json({ok: true}); +} diff --git a/src/pages/api/storage/insert.ts b/src/pages/api/storage/insert.ts new file mode 100644 index 00000000..7baa8196 --- /dev/null +++ b/src/pages/api/storage/insert.ts @@ -0,0 +1,46 @@ +// Next.js API route support: https://nextjs.org/docs/api-routes/introduction +import type {NextApiRequest, NextApiResponse} from "next"; +import {withIronSessionApiRoute} from "iron-session/next"; +import {sessionOptions} from "@/lib/session"; +import axios, {AxiosResponse} from "axios"; +import formidable from "formidable-serverless"; +import {getDownloadURL, ref, uploadBytes} from "firebase/storage"; +import fs from "fs"; +import {app, storage} from "@/firebase"; +import {doc, getDoc, getFirestore, setDoc} from "firebase/firestore"; +import {Stat} from "@/interfaces/user"; +import {speakingReverseMarking} from "@/utils/score"; + +const db = getFirestore(app); +export default withIronSessionApiRoute(handler, sessionOptions); + +async function handler(req: NextApiRequest, res: NextApiResponse) { + if (!req.session.user) { + res.status(401).json({ok: false}); + return; + } + + const form = formidable({keepExtensions: true}); + await form.parse(req, async (err: any, fields: any, files: any) => { + if (err) { + console.log(err); + return res.status(500).json({ok: false}); + } + + const audioFile = files.audio; + console.log({fields}, (audioFile as any).path); + const audioFileRef = ref(storage, `${fields.root}/${(audioFile as any).path.split("/").pop()!.replace("upload_", "")}`); + + const binary = fs.readFileSync((audioFile as any).path).buffer; + const snapshot = await uploadBytes(audioFileRef, binary); + + const path = await getDownloadURL(snapshot.ref); + res.status(200).json({path}); + }); +} + +export const config = { + api: { + bodyParser: false, + }, +}; diff --git a/src/utils/evaluation.ts b/src/utils/evaluation.ts index d9c53b19..932d4e0d 100644 --- a/src/utils/evaluation.ts +++ b/src/utils/evaluation.ts @@ -45,16 +45,17 @@ export const evaluateSpeakingAnswer = async (exercise: SpeakingExercise | Intera } }; -const downloadBlob = async (url: string): Promise => { +export const downloadBlob = async (url: string): Promise => { const blobResponse = await axios.get(url, {responseType: "arraybuffer"}); return Buffer.from(blobResponse.data, "binary"); }; const evaluateSpeakingExercise = async (exercise: SpeakingExercise, exerciseId: string, solution: UserSolution, id: string) => { + const formData = new FormData(); + const audioBlob = await downloadBlob(solution.solutions[0].solution.trim()); const audioFile = new File([audioBlob], "audio.wav", {type: "audio/wav"}); - const formData = new FormData(); formData.append("audio", audioFile, "audio.wav"); const evaluationQuestion = @@ -64,7 +65,7 @@ const evaluateSpeakingExercise = async (exercise: SpeakingExercise, exerciseId: const config = { headers: { - "Content-Type": "audio/mp3", + "Content-Type": "audio/wav", }, };