Mp3 uploading is now done on next, now doing concurrent reading and listening exercise calls with ayncio's gather to openai, should be faster

app/api/listening.py

@@ -49,15 +49,3 @@ async def generate_listening_exercise(
     listening_controller: IListeningController = Depends(Provide[controller])
 ):
     return await listening_controller.get_listening_question(section, dto)
-
-
-@listening_router.post(
-    '/',
-    dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
-)
-@inject
-async def save_listening(
-    data: SaveListeningDTO,
-    listening_controller: IListeningController = Depends(Provide[controller])
-):
-    return await listening_controller.save_listening(data)

app/controllers/abc/listening.py

@@ -15,7 +15,3 @@ class IListeningController(ABC):
     @abstractmethod
     async def generate_mp3(self, dto):
         pass
-
-    @abstractmethod
-    async def save_listening(self, data):
-        pass

app/controllers/impl/listening.py

@@ -1,7 +1,10 @@
+import io
+
+from starlette.responses import StreamingResponse
+
 from app.controllers.abc import IListeningController
-from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises, Dialog
+from app.dtos.listening import GenerateListeningExercises, Dialog
 from app.services.abc import IListeningService
-from fastapi import Response
 
 
 class ListeningController(IListeningController):
@@ -17,13 +20,12 @@ class ListeningController(IListeningController):
 
     async def generate_mp3(self, dto: Dialog):
         mp3 = await self._service.generate_mp3(dto)
-        return Response(
+
-            content=mp3,
+        return StreamingResponse(
+            content=io.BytesIO(mp3),
             media_type="audio/mpeg",
             headers={
+                "Content-Type": "audio/mpeg",
                 "Content-Disposition": "attachment;filename=speech.mp3"
             }
         )
-
-    async def save_listening(self, data: SaveListeningDTO):
-        return await self._service.save_listening(data.parts, data.minTimer, data.difficulty, data.id)

app/services/abc/exam/listening.py

@@ -23,7 +23,3 @@ class IListeningService(ABC):
     @abstractmethod
     async def get_dialog_from_audio(self, upload: UploadFile):
         pass
-
-    @abstractmethod
-    async def save_listening(self, parts: list[dict], min_timer: int, difficulty: str, listening_id: str) -> Dict:
-        pass

app/services/impl/exam/listening/__init__.py

@@ -1,21 +1,18 @@
-import queue
+import asyncio
-import uuid
 from logging import getLogger
-from queue import Queue
 import random
-from typing import Dict, List
+from typing import Dict
 
 from starlette.datastructures import UploadFile
 
-from app.dtos.listening import GenerateListeningExercises, Dialog
+from app.dtos.listening import GenerateListeningExercises, Dialog, ListeningExercises
 from app.repositories.abc import IFileStorage, IDocumentStore
 from app.services.abc import IListeningService, ILLMService, ITextToSpeechService, ISpeechToTextService
-from app.configs.question_templates import getListeningTemplate, getListeningPartTemplate
 from app.configs.constants import (
-    NeuralVoices, GPTModels, TemperatureSettings, FilePaths, MinTimers, ExamVariant, EducationalContent,
+    NeuralVoices, GPTModels, TemperatureSettings, EducationalContent,
     FieldsAndExercises
 )
-from app.helpers import ExercisesHelper, FileHelper
+from app.helpers import FileHelper
 from .multiple_choice import MultipleChoice
 from .write_blank_forms import WriteBlankForms
 from .write_blanks import WriteBlanks
@@ -92,83 +89,63 @@ class ListeningService(IListeningService):
         dialog = await self._stt.speech_to_text(f'./tmp/{path_id}/upload.{ext}')
         FileHelper.remove_directory(f'./tmp/{path_id}')
 
-    async def get_listening_question(self, section: int, dto: GenerateListeningExercises):
-        dialog_type = self._sections[f'section_{section}']["type"]
-
-        exercises = []
-        start_id = 1
-        for req_exercise in dto.exercises:
-            if req_exercise.type == "multipleChoice" or req_exercise.type == "multipleChoice3Options":
-                n_options = 4 if "multipleChoice" else 3
-                question = await self._multiple_choice.gen_multiple_choice(
-                    dialog_type, dto.text, req_exercise.quantity, start_id, dto.difficulty, n_options
-                )
-
-                exercises.append(question)
-                self._logger.info(f"Added multiple choice: {question}")
-
-            elif req_exercise.type == "writeBlanksQuestions":
-                question = await self._write_blanks.gen_write_blanks_questions(
-                    dialog_type, dto.text, req_exercise.quantity, start_id, dto.difficulty
-                )
-                question["variant"] = "questions"
-                exercises.append(question)
-                self._logger.info(f"Added write blanks questions: {question}")
-
-            elif req_exercise.type == "writeBlanksFill":
-                question = await self._write_blanks_notes.gen_write_blanks_notes(
-                    dialog_type, dto.text, req_exercise.quantity, start_id, dto.difficulty
-                )
-                question["variant"] = "fill"
-                exercises.append(question)
-                self._logger.info(f"Added write blanks notes: {question}")
-
-            elif req_exercise.type == "writeBlanksForm":
-                question = await self._write_blanks_forms.gen_write_blanks_form(
-                    dialog_type, dto.text, req_exercise.quantity, start_id, dto.difficulty
-                )
-                question["variant"] = "form"
-                exercises.append(question)
-                self._logger.info(f"Added write blanks form: {question}")
-
-            start_id = start_id + req_exercise.quantity
-
-        return {"exercises": exercises}
-
     async def generate_mp3(self, dto: Dialog) -> bytes:
         return await self._tts.text_to_speech(dto)
 
-    async def save_listening(self, parts: list[dict], min_timer: int, difficulty: str, listening_id: str):
+    async def get_listening_question(self, section: int, dto: GenerateListeningExercises):
-        template = getListeningTemplate()
+        dialog_type = self._sections[f'section_{section}']["type"]
-        template['difficulty'] = difficulty
+        start_id = 1
-        for i, part in enumerate(parts, start=0):
+        exercise_tasks = []
-            part_template = getListeningPartTemplate()
 
-            file_name = str(uuid.uuid4()) + ".mp3"
+        for req_exercise in dto.exercises:
-            sound_file_path = FilePaths.AUDIO_FILES_PATH + file_name
+            exercise_tasks.append(
-            firebase_file_path = FilePaths.FIREBASE_LISTENING_AUDIO_FILES_PATH + file_name
+                self._generate_exercise(
-            if "conversation" in part["text"]:
+                    req_exercise,
-                await self._tts.text_to_speech(part["text"]["conversation"], sound_file_path)
+                    dialog_type,
-            else:
+                    dto.text,
-                await self._tts.text_to_speech(part["text"], sound_file_path)
+                    start_id,
-            file_url = await self._file_storage.upload_file_firebase_get_url(firebase_file_path, sound_file_path)
+                    dto.difficulty
+                )
+            )
+            start_id += req_exercise.quantity
 
-            part_template["audio"]["source"] = file_url
+        return {"exercises": await asyncio.gather(*exercise_tasks) }
-            part_template["exercises"] = part["exercises"]
 
-            template['parts'].append(part_template)
+    async def _generate_exercise(
+            self, req_exercise: ListeningExercises, dialog_type: str, text: str, start_id: int, difficulty: str
+    ):
+        if req_exercise.type == "multipleChoice" or req_exercise.type == "multipleChoice3Options":
+            n_options = 4 if req_exercise.type == "multipleChoice" else 3
+            question = await self._multiple_choice.gen_multiple_choice(
+                dialog_type, text, req_exercise.quantity, start_id, difficulty, n_options
+            )
+            self._logger.info(f"Added multiple choice: {question}")
+            return question
 
-        if min_timer != MinTimers.LISTENING_MIN_TIMER_DEFAULT:
+        elif req_exercise.type == "writeBlanksQuestions":
-            template["minTimer"] = min_timer
+            question = await self._write_blanks.gen_write_blanks_questions(
-            template["variant"] = ExamVariant.PARTIAL.value
+                dialog_type, text, req_exercise.quantity, start_id, difficulty
-        else:
+            )
-            template["variant"] = ExamVariant.FULL.value
+            question["variant"] = "questions"
+            self._logger.info(f"Added write blanks questions: {question}")
+            return question
+
+        elif req_exercise.type == "writeBlanksFill":
+            question = await self._write_blanks_notes.gen_write_blanks_notes(
+                dialog_type, text, req_exercise.quantity, start_id, difficulty
+            )
+            question["variant"] = "fill"
+            self._logger.info(f"Added write blanks notes: {question}")
+            return question
+
+        elif req_exercise.type == "writeBlanksForm":
+            question = await self._write_blanks_forms.gen_write_blanks_form(
+                dialog_type, text, req_exercise.quantity, start_id, difficulty
+            )
+            question["variant"] = "form"
+            self._logger.info(f"Added write blanks form: {question}")
+            return question
 
-        listening_id = await self._document_store.save_to_db("listening", template, listening_id)
-        if listening_id:
-            return {**template, "id": listening_id}
-        else:
-            raise Exception("Failed to save question: " + str(parts))
 
     # ==================================================================================================================
     # generate_listening_question helpers

app/services/impl/exam/reading/__init__.py

@@ -1,3 +1,4 @@
+import asyncio
 from logging import getLogger
 
 from fastapi import UploadFile
@@ -77,55 +78,63 @@ class ReadingService(IReadingService):
             TemperatureSettings.GEN_QUESTION_TEMPERATURE
         )
 
-    async def generate_reading_exercises(self, dto: ReadingDTO):
+    async def _generate_single_exercise(self, req_exercise, text: str, start_id: int, difficulty: str) -> dict:
-        exercises = []
-        start_id = 1
-        for req_exercise in dto.exercises:
         if req_exercise.type == "fillBlanks":
             question = await self._fill_blanks.gen_summary_fill_blanks_exercise(
-                    dto.text, req_exercise.quantity, start_id, dto.difficulty, req_exercise.num_random_words
+                text, req_exercise.quantity, start_id, difficulty, req_exercise.num_random_words
             )
-                exercises.append(question)
             self._logger.info(f"Added fill blanks: {question}")
+            return question
 
         elif req_exercise.type == "trueFalse":
             question = await self._true_false.gen_true_false_not_given_exercise(
-                    dto.text, req_exercise.quantity, start_id, dto.difficulty
+                text, req_exercise.quantity, start_id, difficulty
             )
-                exercises.append(question)
             self._logger.info(f"Added trueFalse: {question}")
+            return question
 
         elif req_exercise.type == "writeBlanks":
             question = await self._write_blanks.gen_write_blanks_exercise(
-                    dto.text, req_exercise.quantity, start_id, dto.difficulty, req_exercise.max_words
+                text, req_exercise.quantity, start_id, difficulty, req_exercise.max_words
             )
 
             if ExercisesHelper.answer_word_limit_ok(question):
-                    exercises.append(question)
                 self._logger.info(f"Added write blanks: {question}")
+                return question
             else:
-                    exercises.append({})
                 self._logger.info("Did not add write blanks because it did not respect word limit")
+                return {}
 
         elif req_exercise.type == "paragraphMatch":
-
             question = await self._paragraph_match.gen_paragraph_match_exercise(
-                    dto.text, req_exercise.quantity, start_id
+                text, req_exercise.quantity, start_id
             )
-                exercises.append(question)
             self._logger.info(f"Added paragraph match: {question}")
+            return question
 
         elif req_exercise.type == "ideaMatch":
-
             question = await self._idea_match.gen_idea_match_exercise(
-                    dto.text, req_exercise.quantity, start_id
+                text, req_exercise.quantity, start_id
             )
             question["variant"] = "ideaMatch"
-                exercises.append(question)
             self._logger.info(f"Added idea match: {question}")
+            return question
 
-            start_id = start_id + req_exercise.quantity
+    async def generate_reading_exercises(self, dto: ReadingDTO):
+        exercise_tasks = []
+        start_id = 1
+
+        for req_exercise in dto.exercises:
+            exercise_tasks.append(
+                self._generate_single_exercise(
+                    req_exercise,
+                    dto.text,
+                    start_id,
+                    dto.difficulty
+                )
+            )
+            start_id += req_exercise.quantity
 
         return {
-            "exercises": exercises
+            "exercises": await asyncio.gather(*exercise_tasks)
         }