From a2e96f8e543f07d044434f727a0af7ad1ee7c2c5 Mon Sep 17 00:00:00 2001 From: Carlos-Mesquita Date: Wed, 6 Nov 2024 11:01:39 +0000 Subject: [PATCH] Batch import wasn't updated --- app/api/listening.py | 14 ++++- app/configs/dependency_injection.py | 10 +++- app/controllers/abc/listening.py | 4 ++ app/controllers/impl/listening.py | 15 ++++- app/controllers/impl/user.py | 2 +- app/dtos/listening.py | 12 +++- app/dtos/user_batch.py | 4 +- app/repositories/abc/document_store.py | 11 ++-- .../impl/document_stores/firestore.py | 5 +- .../impl/document_stores/mongo.py | 8 ++- app/server.py | 4 +- app/services/abc/exam/listening.py | 4 ++ app/services/abc/third_parties/tts.py | 2 +- app/services/abc/user.py | 2 +- app/services/impl/exam/listening/__init__.py | 5 +- app/services/impl/third_parties/aws_polly.py | 33 +++++------ app/services/impl/training/training.py | 10 ++-- app/services/impl/user.py | 57 +++++++------------ 18 files changed, 124 insertions(+), 78 deletions(-) diff --git a/app/api/listening.py b/app/api/listening.py index 6636355..4cc9dc4 100644 --- a/app/api/listening.py +++ b/app/api/listening.py @@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, Path, Query from app.middlewares import Authorized, IsAuthenticatedViaBearerToken from app.controllers.abc import IListeningController from app.configs.constants import EducationalContent, ListeningExerciseType -from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises +from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises, Dialog controller = "listening_controller" listening_router = APIRouter() @@ -26,6 +26,18 @@ async def generate_listening_dialog( topic = random.choice(EducationalContent.TOPICS) if not topic else topic return await listening_controller.generate_listening_dialog(section, difficulty, topic) +@listening_router.post( + '/media', + dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] +) +@inject +async def generate_mp3( + dto: Dialog, + listening_controller: IListeningController = Depends(Provide[controller]) +): + return await listening_controller.generate_mp3(dto) + + @listening_router.post( '/{section}', dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] diff --git a/app/configs/dependency_injection.py b/app/configs/dependency_injection.py index 121f3bd..deb9626 100644 --- a/app/configs/dependency_injection.py +++ b/app/configs/dependency_injection.py @@ -110,7 +110,11 @@ class DependencyInjector: self._container.training_service = providers.Factory( TrainingService, llm=self._container.llm, - firestore=self._container.document_store, training_kb=self._container.training_kb + document_store=self._container.document_store, training_kb=self._container.training_kb + ) + + self._container.user_service = providers.Factory( + UserService, document_store=self._container.document_store ) def _setup_controllers(self): @@ -120,6 +124,10 @@ class DependencyInjector: writing_service=self._container.writing_service ) + self._container.user_controller = providers.Factory( + UserController, user_service=self._container.user_service + ) + self._container.training_controller = providers.Factory( TrainingController, training_service=self._container.training_service ) diff --git a/app/controllers/abc/listening.py b/app/controllers/abc/listening.py index 5e07a50..3190ba1 100644 --- a/app/controllers/abc/listening.py +++ b/app/controllers/abc/listening.py @@ -12,6 +12,10 @@ class IListeningController(ABC): async def get_listening_question(self, section: int, dto): pass + @abstractmethod + async def generate_mp3(self, dto): + pass + @abstractmethod async def save_listening(self, data): pass diff --git a/app/controllers/impl/listening.py b/app/controllers/impl/listening.py index 8102d71..ed1f38e 100644 --- a/app/controllers/impl/listening.py +++ b/app/controllers/impl/listening.py @@ -1,8 +1,7 @@ -from typing import List - from app.controllers.abc import IListeningController -from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises +from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises, Dialog from app.services.abc import IListeningService +from fastapi import Response class ListeningController(IListeningController): @@ -16,5 +15,15 @@ class ListeningController(IListeningController): async def get_listening_question(self, section: int, dto: GenerateListeningExercises): return await self._service.get_listening_question(section, dto) + async def generate_mp3(self, dto: Dialog): + mp3 = await self._service.generate_mp3(dto) + return Response( + content=mp3, + media_type="audio/mpeg", + headers={ + "Content-Disposition": "attachment;filename=speech.mp3" + } + ) + async def save_listening(self, data: SaveListeningDTO): return await self._service.save_listening(data.parts, data.minTimer, data.difficulty, data.id) diff --git a/app/controllers/impl/user.py b/app/controllers/impl/user.py index 9952fb8..ffdbbec 100644 --- a/app/controllers/impl/user.py +++ b/app/controllers/impl/user.py @@ -9,4 +9,4 @@ class UserController(IUserController): self._service = user_service async def batch_import(self, batch: BatchUsersDTO): - return await self._service.fetch_tips(batch) + return await self._service.batch_users(batch) diff --git a/app/dtos/listening.py b/app/dtos/listening.py index 81cc500..8c98434 100644 --- a/app/dtos/listening.py +++ b/app/dtos/listening.py @@ -2,7 +2,7 @@ import random import uuid from typing import List, Dict, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field from app.configs.constants import MinTimers, EducationalContent, ListeningExerciseType @@ -22,3 +22,13 @@ class GenerateListeningExercises(BaseModel): text: str exercises: List[ListeningExercises] difficulty: Optional[str] + +class ConversationPayload(BaseModel): + name: str + gender: str + text: str + voice: str + +class Dialog(BaseModel): + conversation: Optional[List[ConversationPayload]] = Field(default_factory=list) + monologue: Optional[str] = None diff --git a/app/dtos/user_batch.py b/app/dtos/user_batch.py index a44778b..2198f5f 100644 --- a/app/dtos/user_batch.py +++ b/app/dtos/user_batch.py @@ -10,8 +10,8 @@ class DemographicInfo(BaseModel): country: Optional[str] = None class Entity(BaseModel): - id: str - role: str + id: str + role: str class UserDTO(BaseModel): diff --git a/app/repositories/abc/document_store.py b/app/repositories/abc/document_store.py index 03c041f..3443eea 100644 --- a/app/repositories/abc/document_store.py +++ b/app/repositories/abc/document_store.py @@ -5,11 +5,14 @@ from typing import Dict, Optional, List class IDocumentStore(ABC): - async def save_to_db(self, collection: str, item: Dict, doc_id: Optional[str]) -> Optional[str]: - pass - - async def get_all(self, collection: str) -> List[Dict]: + async def save_to_db(self, collection: str, item: Dict, doc_id: Optional[str] = None) -> Optional[str]: pass async def get_doc_by_id(self, collection: str, doc_id: str) -> Optional[Dict]: pass + + async def find(self, collection: str, query: Optional[Dict]) -> List[Dict]: + pass + + async def update(self, collection: str, filter_query: Dict, update: Dict) -> Optional[str]: + pass diff --git a/app/repositories/impl/document_stores/firestore.py b/app/repositories/impl/document_stores/firestore.py index db7c4cb..2facebb 100644 --- a/app/repositories/impl/document_stores/firestore.py +++ b/app/repositories/impl/document_stores/firestore.py @@ -30,7 +30,7 @@ class Firestore(IDocumentStore): return None - async def get_all(self, collection: str) -> List[Dict]: + async def find(self, collection: str, query: Optional[Dict] = None) -> List[Dict]: collection_ref: AsyncCollectionReference = self._client.collection(collection) docs = [] async for doc in collection_ref.stream(): @@ -45,3 +45,6 @@ class Firestore(IDocumentStore): if doc.exists: return doc.to_dict() return None + + async def update(self, collection: str, filter_query: Dict, update: Dict) -> Optional[str]: + raise NotImplemented() diff --git a/app/repositories/impl/document_stores/mongo.py b/app/repositories/impl/document_stores/mongo.py index c1f1097..c561ec6 100644 --- a/app/repositories/impl/document_stores/mongo.py +++ b/app/repositories/impl/document_stores/mongo.py @@ -29,9 +29,13 @@ class MongoDB(IDocumentStore): return None - async def get_all(self, collection: str) -> List[Dict]: - cursor = self._mongo_db[collection].find() + async def find(self, collection: str, query: Optional[Dict] = None) -> List[Dict]: + query = query if query else {} + cursor = self._mongo_db[collection].find(query) return [document async for document in cursor] + async def update(self, collection: str, filter_query: Dict, update: Dict) -> Optional[str]: + return (await self._mongo_db[collection].update_one(filter_query, update)).upserted_id + async def get_doc_by_id(self, collection: str, doc_id: str) -> Optional[Dict]: return await self._mongo_db[collection].find_one({"id": doc_id}) diff --git a/app/server.py b/app/server.py index 96ae087..a064fa3 100644 --- a/app/server.py +++ b/app/server.py @@ -50,8 +50,8 @@ async def lifespan(_app: FastAPI): session.client( 'polly', region_name='eu-west-1', - aws_secret_access_key=os.getenv("AWS_ACCESS_KEY_ID"), - aws_access_key_id=os.getenv("AWS_SECRET_ACCESS_KEY") + aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID") ) ) diff --git a/app/services/abc/exam/listening.py b/app/services/abc/exam/listening.py index 0f69a10..bde4ecd 100644 --- a/app/services/abc/exam/listening.py +++ b/app/services/abc/exam/listening.py @@ -16,6 +16,10 @@ class IListeningService(ABC): async def get_listening_question(self, section: int, dto): pass + @abstractmethod + async def generate_mp3(self, dto) -> bytes: + pass + @abstractmethod async def get_dialog_from_audio(self, upload: UploadFile): pass diff --git a/app/services/abc/third_parties/tts.py b/app/services/abc/third_parties/tts.py index c28cd42..018cc26 100644 --- a/app/services/abc/third_parties/tts.py +++ b/app/services/abc/third_parties/tts.py @@ -9,7 +9,7 @@ class ITextToSpeechService(ABC): pass @abstractmethod - async def text_to_speech(self, text: Union[list[str], str], file_name: str): + async def text_to_speech(self, dialog) -> bytes: pass @abstractmethod diff --git a/app/services/abc/user.py b/app/services/abc/user.py index 2472f5c..10f4886 100644 --- a/app/services/abc/user.py +++ b/app/services/abc/user.py @@ -6,5 +6,5 @@ from app.dtos.user_batch import BatchUsersDTO class IUserService(ABC): @abstractmethod - async def fetch_tips(self, batch: BatchUsersDTO): + async def batch_users(self, batch: BatchUsersDTO): pass diff --git a/app/services/impl/exam/listening/__init__.py b/app/services/impl/exam/listening/__init__.py index 13a5aa2..d16265a 100644 --- a/app/services/impl/exam/listening/__init__.py +++ b/app/services/impl/exam/listening/__init__.py @@ -7,7 +7,7 @@ from typing import Dict, List from starlette.datastructures import UploadFile -from app.dtos.listening import GenerateListeningExercises +from app.dtos.listening import GenerateListeningExercises, Dialog from app.repositories.abc import IFileStorage, IDocumentStore from app.services.abc import IListeningService, ILLMService, ITextToSpeechService, ISpeechToTextService from app.configs.question_templates import getListeningTemplate, getListeningPartTemplate @@ -135,6 +135,9 @@ class ListeningService(IListeningService): return {"exercises": exercises} + async def generate_mp3(self, dto: Dialog) -> bytes: + return await self._tts.text_to_speech(dto) + async def save_listening(self, parts: list[dict], min_timer: int, difficulty: str, listening_id: str): template = getListeningTemplate() template['difficulty'] = difficulty diff --git a/app/services/impl/third_parties/aws_polly.py b/app/services/impl/third_parties/aws_polly.py index 559e8d1..36af75f 100644 --- a/app/services/impl/third_parties/aws_polly.py +++ b/app/services/impl/third_parties/aws_polly.py @@ -4,6 +4,7 @@ from typing import Union import aiofiles from aiobotocore.client import BaseClient +from app.dtos.listening import Dialog from app.services.abc import ITextToSpeechService from app.configs.constants import NeuralVoices @@ -22,14 +23,15 @@ class AWSPolly(ITextToSpeechService): ) return await tts_response['AudioStream'].read() - async def text_to_speech(self, text: Union[list[str], str], file_name: str): - if isinstance(text, str): - audio_segments = await self._text_to_speech(text) - elif isinstance(text, list): - audio_segments = await self._conversation_to_speech(text) - else: + async def text_to_speech(self, dialog: Dialog) -> bytes: + if not dialog.conversation and not dialog.monologue: raise ValueError("Unsupported argument for text_to_speech") + if not dialog.conversation: + audio_segments = await self._text_to_speech(dialog.monologue) + else: + audio_segments = await self._conversation_to_speech(dialog) + final_message = await self.synthesize_speech( "This audio recording, for the listening exercise, has finished.", "Stephen" @@ -40,27 +42,26 @@ class AWSPolly(ITextToSpeechService): # Combine the audio segments into a single audio file combined_audio = b"".join(audio_segments) - # Save the combined audio to a single file - async with aiofiles.open(file_name, "wb") as f: - await f.write(combined_audio) - print("Speech segments saved to " + file_name) + return combined_audio + # Save the combined audio to a single file + #async with aiofiles.open(file_name, "wb") as f: + # await f.write(combined_audio) + + #print("Speech segments saved to " + file_name) async def _text_to_speech(self, text: str): voice = random.choice(NeuralVoices.ALL_NEURAL_VOICES)['Id'] - # Initialize an empty list to store audio segments audio_segments = [] for part in self._divide_text(text): audio_segments.append(await self.synthesize_speech(part, voice)) return audio_segments - async def _conversation_to_speech(self, conversation: list): - # Initialize an empty list to store audio segments + async def _conversation_to_speech(self, dialog: Dialog): audio_segments = [] - # Iterate through the text segments, convert to audio segments, and store them - for segment in conversation: - audio_segments.append(await self.synthesize_speech(segment["text"], segment["voice"])) + for convo_payload in dialog.conversation: + audio_segments.append(await self.synthesize_speech(convo_payload.text, convo_payload.voice)) return audio_segments diff --git a/app/services/impl/training/training.py b/app/services/impl/training/training.py index 687af2c..87fe007 100644 --- a/app/services/impl/training/training.py +++ b/app/services/impl/training/training.py @@ -1,4 +1,5 @@ import re +import uuid from datetime import datetime from functools import reduce from logging import getLogger @@ -23,9 +24,9 @@ class TrainingService(ITrainingService): ] # strategy word_link ct_focus reading_skill word_partners writing_skill language_for_writing - def __init__(self, llm: ILLMService, firestore: IDocumentStore, training_kb: IKnowledgeBase): + def __init__(self, llm: ILLMService, document_store: IDocumentStore, training_kb: IKnowledgeBase): self._llm = llm - self._db = firestore + self._db = document_store self._kb = training_kb self._logger = getLogger(__name__) @@ -96,16 +97,15 @@ class TrainingService(ITrainingService): for area in training_content.weak_areas: weak_areas["weak_areas"].append(area.dict()) - new_id = str(uuid.uuid4()) training_doc = { - 'id': new_id, 'created_at': int(datetime.now().timestamp() * 1000), **exam_map, **usefull_tips.dict(), **weak_areas, "user": user } - doc_id = await self._db.save_to_db('training', training_doc) + new_id = await self._db.save_to_db('training', training_doc) + return { "id": new_id } diff --git a/app/services/impl/user.py b/app/services/impl/user.py index 631f335..3cc2f6e 100644 --- a/app/services/impl/user.py +++ b/app/services/impl/user.py @@ -2,20 +2,17 @@ import os import subprocess import time import uuid -import pandas as pd -import shortuuid from datetime import datetime from logging import getLogger import pandas as pd -from typing import Dict import shortuuid -from pymongo.database import Database from app.dtos.user_batch import BatchUsersDTO, UserDTO from app.helpers import FileHelper +from app.repositories.abc import IDocumentStore from app.services.abc import IUserService @@ -34,14 +31,14 @@ class UserService(IUserService): "speaking": 0, } - def __init__(self, mongo: Database): - self._db: Database = mongo + def __init__(self, document_store: IDocumentStore): + self._db = document_store self._logger = getLogger(__name__) - def fetch_tips(self, batch: BatchUsersDTO): + def batch_users(self, batch_dto: BatchUsersDTO): file_name = f'{uuid.uuid4()}.csv' path = f'./tmp/{file_name}' - self._generate_firebase_auth_csv(batch, path) + self._generate_firebase_auth_csv(batch_dto, path) result = self._upload_users('./tmp', file_name) if result.returncode != 0: @@ -49,20 +46,11 @@ class UserService(IUserService): self._logger.error(error_msg) return error_msg - self._init_users(batch) + self._init_users(batch_dto) FileHelper.remove_file(path) return {"ok": True} - @staticmethod - def _map_to_batch(request_data: Dict) -> BatchUsersDTO: - users_list = [{**user} for user in request_data["users"]] - for user in users_list: - user["studentID"] = str(user["studentID"]) - - users: list[UserDTO] = [UserDTO(**user) for user in users_list] - return BatchUsersDTO(makerID=request_data["makerID"], users=users) - @staticmethod def _generate_firebase_auth_csv(batch_dto: BatchUsersDTO, path: str): # https://firebase.google.com/docs/cli/auth#file_format @@ -127,22 +115,21 @@ class UserService(IUserService): result = subprocess.run(command, shell=True, cwd=directory, capture_output=True, text=True) return result - def _init_users(self, batch_users: BatchUsersDTO): + async def _init_users(self, batch_users: BatchUsersDTO): maker_id = batch_users.makerID for user in batch_users.users: - self._insert_new_user(user) - code = self._create_code(user, maker_id) + await self._insert_new_user(user) + await self._create_code(user, maker_id) if user.groupName and len(user.groupName.strip()) > 0: - self._assign_user_to_group_by_name(user, maker_id) + await self._assign_user_to_group_by_name(user, maker_id) - def _insert_new_user(self, user: UserDTO): + async def _insert_new_user(self, user: UserDTO): new_user = { **user.dict(exclude={ 'passport_id', 'groupName', 'expiryDate', 'corporate', 'passwordHash', 'passwordSalt' }), - 'id': str(user.id), 'bio': "", 'focus': "academic", 'status': "active", @@ -155,11 +142,11 @@ class UserService(IUserService): 'subscriptionExpirationDate': user.expiryDate, 'entities': user.entities } - self._db.users.insert_one(new_user) + await self._db.save_to_db("users", new_user, str(user.id)) - def _create_code(self, user: UserDTO, maker_id: str) -> str: + async def _create_code(self, user: UserDTO, maker_id: str) -> str: code = shortuuid.ShortUUID().random(length=6) - self._db.codes.insert_one({ + await self._db.save_to_db("codes", { 'id': code, 'code': code, 'creator': maker_id, @@ -170,34 +157,32 @@ class UserService(IUserService): 'email': user.email, 'name': user.name, 'passport_id': user.passport_id - }) + }, code) return code - def _assign_user_to_group_by_name(self, user: UserDTO, maker_id: str): + async def _assign_user_to_group_by_name(self, user: UserDTO, maker_id: str): user_id = str(user.id) - groups = list(self._db.groups.find( - { + groups = await self._db.find("groups", { "admin": maker_id, "name": user.groupName.strip() - } - )) + }) if len(groups) == 0: new_group = { - 'id': str(uuid.uuid4()), 'admin': maker_id, 'name': user.groupName.strip(), 'participants': [user_id], 'disableEditing': False, } - self._db.groups.insert_one(new_group) + await self._db.save_to_db("groups", new_group, str(uuid.uuid4())) else: group = groups[0] participants = group["participants"] if user_id not in participants: participants.append(user_id) - self._db.groups.update_one( + await self._db.update( + "groups", {"id": group["id"]}, {"$set": {"participants": participants}} )