devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Batch import wasn't updated

Browse Source
This commit is contained in:
Carlos-Mesquita
2024-11-06 11:01:39 +00:00
parent e51cd891d2
commit a2e96f8e54
18 changed files with 124 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
app/api/listening.py
View File

@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, Path, Query
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
from app.controllers.abc import IListeningController
from app.configs.constants import EducationalContent, ListeningExerciseType
from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises
from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises, Dialog
controller = "listening_controller"
listening_router = APIRouter()
@@ -26,6 +26,18 @@ async def generate_listening_dialog(
topic = random.choice(EducationalContent.TOPICS) if not topic else topic
return await listening_controller.generate_listening_dialog(section, difficulty, topic)
@listening_router.post(
'/media',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
)
@inject
async def generate_mp3(
dto: Dialog,
listening_controller: IListeningController = Depends(Provide[controller])
):
return await listening_controller.generate_mp3(dto)
@listening_router.post(
'/{section}',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]

10
app/configs/dependency_injection.py
View File

@@ -110,7 +110,11 @@ class DependencyInjector:
self._container.training_service = providers.Factory(
TrainingService, llm=self._container.llm,
firestore=self._container.document_store, training_kb=self._container.training_kb
document_store=self._container.document_store, training_kb=self._container.training_kb
)
self._container.user_service = providers.Factory(
UserService, document_store=self._container.document_store
)
def _setup_controllers(self):
@@ -120,6 +124,10 @@ class DependencyInjector:
writing_service=self._container.writing_service
)
self._container.user_controller = providers.Factory(
UserController, user_service=self._container.user_service
)
self._container.training_controller = providers.Factory(
TrainingController, training_service=self._container.training_service
)

4
app/controllers/abc/listening.py
View File

@@ -12,6 +12,10 @@ class IListeningController(ABC):
async def get_listening_question(self, section: int, dto):
pass
@abstractmethod
async def generate_mp3(self, dto):
pass
@abstractmethod
async def save_listening(self, data):
pass

15
app/controllers/impl/listening.py
View File

@@ -1,8 +1,7 @@
from typing import List
from app.controllers.abc import IListeningController
from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises
from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises, Dialog
from app.services.abc import IListeningService
from fastapi import Response
class ListeningController(IListeningController):
@@ -16,5 +15,15 @@ class ListeningController(IListeningController):
async def get_listening_question(self, section: int, dto: GenerateListeningExercises):
return await self._service.get_listening_question(section, dto)
async def generate_mp3(self, dto: Dialog):
mp3 = await self._service.generate_mp3(dto)
return Response(
content=mp3,
media_type="audio/mpeg",
headers={
"Content-Disposition": "attachment;filename=speech.mp3"
}
)
async def save_listening(self, data: SaveListeningDTO):
return await self._service.save_listening(data.parts, data.minTimer, data.difficulty, data.id)

2
app/controllers/impl/user.py
View File

@@ -9,4 +9,4 @@ class UserController(IUserController):
self._service = user_service
async def batch_import(self, batch: BatchUsersDTO):
return await self._service.fetch_tips(batch)
return await self._service.batch_users(batch)

12
app/dtos/listening.py
View File

@@ -2,7 +2,7 @@ import random
import uuid
from typing import List, Dict, Optional
from pydantic import BaseModel
from pydantic import BaseModel, Field
from app.configs.constants import MinTimers, EducationalContent, ListeningExerciseType
@@ -22,3 +22,13 @@ class GenerateListeningExercises(BaseModel):
text: str
exercises: List[ListeningExercises]
difficulty: Optional[str]
class ConversationPayload(BaseModel):
name: str
gender: str
text: str
voice: str
class Dialog(BaseModel):
conversation: Optional[List[ConversationPayload]] = Field(default_factory=list)
monologue: Optional[str] = None

11
app/repositories/abc/document_store.py
View File

@@ -5,11 +5,14 @@ from typing import Dict, Optional, List
class IDocumentStore(ABC):
async def save_to_db(self, collection: str, item: Dict, doc_id: Optional[str]) -> Optional[str]:
pass
async def get_all(self, collection: str) -> List[Dict]:
async def save_to_db(self, collection: str, item: Dict, doc_id: Optional[str] = None) -> Optional[str]:
pass
async def get_doc_by_id(self, collection: str, doc_id: str) -> Optional[Dict]:
pass
async def find(self, collection: str, query: Optional[Dict]) -> List[Dict]:
pass
async def update(self, collection: str, filter_query: Dict, update: Dict) -> Optional[str]:
pass

5
app/repositories/impl/document_stores/firestore.py
View File

@@ -30,7 +30,7 @@ class Firestore(IDocumentStore):
return None
async def get_all(self, collection: str) -> List[Dict]:
async def find(self, collection: str, query: Optional[Dict] = None) -> List[Dict]:
collection_ref: AsyncCollectionReference = self._client.collection(collection)
docs = []
async for doc in collection_ref.stream():
@@ -45,3 +45,6 @@ class Firestore(IDocumentStore):
if doc.exists:
return doc.to_dict()
return None
async def update(self, collection: str, filter_query: Dict, update: Dict) -> Optional[str]:
raise NotImplemented()

8
app/repositories/impl/document_stores/mongo.py
View File

@@ -29,9 +29,13 @@ class MongoDB(IDocumentStore):
return None
async def get_all(self, collection: str) -> List[Dict]:
cursor = self._mongo_db[collection].find()
async def find(self, collection: str, query: Optional[Dict] = None) -> List[Dict]:
query = query if query else {}
cursor = self._mongo_db[collection].find(query)
return [document async for document in cursor]
async def update(self, collection: str, filter_query: Dict, update: Dict) -> Optional[str]:
return (await self._mongo_db[collection].update_one(filter_query, update)).upserted_id
async def get_doc_by_id(self, collection: str, doc_id: str) -> Optional[Dict]:
return await self._mongo_db[collection].find_one({"id": doc_id})

4
app/server.py
View File

@@ -50,8 +50,8 @@ async def lifespan(_app: FastAPI):
session.client(
'polly',
region_name='eu-west-1',
aws_secret_access_key=os.getenv("AWS_ACCESS_KEY_ID"),
aws_access_key_id=os.getenv("AWS_SECRET_ACCESS_KEY")
aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID")
)
)

4
app/services/abc/exam/listening.py
View File

@@ -16,6 +16,10 @@ class IListeningService(ABC):
async def get_listening_question(self, section: int, dto):
pass
@abstractmethod
async def generate_mp3(self, dto) -> bytes:
pass
@abstractmethod
async def get_dialog_from_audio(self, upload: UploadFile):
pass

2
app/services/abc/third_parties/tts.py
View File

@@ -9,7 +9,7 @@ class ITextToSpeechService(ABC):
pass
@abstractmethod
async def text_to_speech(self, text: Union[list[str], str], file_name: str):
async def text_to_speech(self, dialog) -> bytes:
pass
@abstractmethod

2
app/services/abc/user.py
View File

@@ -6,5 +6,5 @@ from app.dtos.user_batch import BatchUsersDTO
class IUserService(ABC):
@abstractmethod
async def fetch_tips(self, batch: BatchUsersDTO):
async def batch_users(self, batch: BatchUsersDTO):
pass

5
app/services/impl/exam/listening/__init__.py
View File

@@ -7,7 +7,7 @@ from typing import Dict, List
from starlette.datastructures import UploadFile
from app.dtos.listening import GenerateListeningExercises
from app.dtos.listening import GenerateListeningExercises, Dialog
from app.repositories.abc import IFileStorage, IDocumentStore
from app.services.abc import IListeningService, ILLMService, ITextToSpeechService, ISpeechToTextService
from app.configs.question_templates import getListeningTemplate, getListeningPartTemplate
@@ -135,6 +135,9 @@ class ListeningService(IListeningService):
return {"exercises": exercises}
async def generate_mp3(self, dto: Dialog) -> bytes:
return await self._tts.text_to_speech(dto)
async def save_listening(self, parts: list[dict], min_timer: int, difficulty: str, listening_id: str):
template = getListeningTemplate()
template['difficulty'] = difficulty

33
app/services/impl/third_parties/aws_polly.py
View File

@@ -4,6 +4,7 @@ from typing import Union
import aiofiles
from aiobotocore.client import BaseClient
from app.dtos.listening import Dialog
from app.services.abc import ITextToSpeechService
from app.configs.constants import NeuralVoices
@@ -22,14 +23,15 @@ class AWSPolly(ITextToSpeechService):
)
return await tts_response['AudioStream'].read()
async def text_to_speech(self, text: Union[list[str], str], file_name: str):
if isinstance(text, str):
audio_segments = await self._text_to_speech(text)
elif isinstance(text, list):
audio_segments = await self._conversation_to_speech(text)
else:
async def text_to_speech(self, dialog: Dialog) -> bytes:
if not dialog.conversation and not dialog.monologue:
raise ValueError("Unsupported argument for text_to_speech")
if not dialog.conversation:
audio_segments = await self._text_to_speech(dialog.monologue)
else:
audio_segments = await self._conversation_to_speech(dialog)
final_message = await self.synthesize_speech(
"This audio recording, for the listening exercise, has finished.",
"Stephen"
@@ -40,27 +42,26 @@ class AWSPolly(ITextToSpeechService):
# Combine the audio segments into a single audio file
combined_audio = b"".join(audio_segments)
# Save the combined audio to a single file
async with aiofiles.open(file_name, "wb") as f:
await f.write(combined_audio)
print("Speech segments saved to " + file_name)
return combined_audio
# Save the combined audio to a single file
#async with aiofiles.open(file_name, "wb") as f:
# await f.write(combined_audio)
#print("Speech segments saved to " + file_name)
async def _text_to_speech(self, text: str):
voice = random.choice(NeuralVoices.ALL_NEURAL_VOICES)['Id']
# Initialize an empty list to store audio segments
audio_segments = []
for part in self._divide_text(text):
audio_segments.append(await self.synthesize_speech(part, voice))
return audio_segments
async def _conversation_to_speech(self, conversation: list):
# Initialize an empty list to store audio segments
async def _conversation_to_speech(self, dialog: Dialog):
audio_segments = []
# Iterate through the text segments, convert to audio segments, and store them
for segment in conversation:
audio_segments.append(await self.synthesize_speech(segment["text"], segment["voice"]))
for convo_payload in dialog.conversation:
audio_segments.append(await self.synthesize_speech(convo_payload.text, convo_payload.voice))
return audio_segments

10
app/services/impl/training/training.py
View File

@@ -1,4 +1,5 @@
import re
import uuid
from datetime import datetime
from functools import reduce
from logging import getLogger
@@ -23,9 +24,9 @@ class TrainingService(ITrainingService):
]
# strategy word_link ct_focus reading_skill word_partners writing_skill language_for_writing
def __init__(self, llm: ILLMService, firestore: IDocumentStore, training_kb: IKnowledgeBase):
def __init__(self, llm: ILLMService, document_store: IDocumentStore, training_kb: IKnowledgeBase):
self._llm = llm
self._db = firestore
self._db = document_store
self._kb = training_kb
self._logger = getLogger(__name__)
@@ -96,16 +97,15 @@ class TrainingService(ITrainingService):
for area in training_content.weak_areas:
weak_areas["weak_areas"].append(area.dict())
new_id = str(uuid.uuid4())
training_doc = {
'id': new_id,
'created_at': int(datetime.now().timestamp() * 1000),
**exam_map,
**usefull_tips.dict(),
**weak_areas,
"user": user
}
doc_id = await self._db.save_to_db('training', training_doc)
new_id = await self._db.save_to_db('training', training_doc)
return {
"id": new_id
}

57
app/services/impl/user.py
View File

@@ -2,20 +2,17 @@ import os
import subprocess
import time
import uuid
import pandas as pd
import shortuuid
from datetime import datetime
from logging import getLogger
import pandas as pd
from typing import Dict
import shortuuid
from pymongo.database import Database
from app.dtos.user_batch import BatchUsersDTO, UserDTO
from app.helpers import FileHelper
from app.repositories.abc import IDocumentStore
from app.services.abc import IUserService
@@ -34,14 +31,14 @@ class UserService(IUserService):
"speaking": 0,
}
def __init__(self, mongo: Database):
self._db: Database = mongo
def __init__(self, document_store: IDocumentStore):
self._db = document_store
self._logger = getLogger(__name__)
def fetch_tips(self, batch: BatchUsersDTO):
def batch_users(self, batch_dto: BatchUsersDTO):
file_name = f'{uuid.uuid4()}.csv'
path = f'./tmp/{file_name}'
self._generate_firebase_auth_csv(batch, path)
self._generate_firebase_auth_csv(batch_dto, path)
result = self._upload_users('./tmp', file_name)
if result.returncode != 0:
@@ -49,20 +46,11 @@ class UserService(IUserService):
self._logger.error(error_msg)
return error_msg
self._init_users(batch)
self._init_users(batch_dto)
FileHelper.remove_file(path)
return {"ok": True}
@staticmethod
def _map_to_batch(request_data: Dict) -> BatchUsersDTO:
users_list = [{**user} for user in request_data["users"]]
for user in users_list:
user["studentID"] = str(user["studentID"])
users: list[UserDTO] = [UserDTO(**user) for user in users_list]
return BatchUsersDTO(makerID=request_data["makerID"], users=users)
@staticmethod
def _generate_firebase_auth_csv(batch_dto: BatchUsersDTO, path: str):
# https://firebase.google.com/docs/cli/auth#file_format
@@ -127,22 +115,21 @@ class UserService(IUserService):
result = subprocess.run(command, shell=True, cwd=directory, capture_output=True, text=True)
return result
def _init_users(self, batch_users: BatchUsersDTO):
async def _init_users(self, batch_users: BatchUsersDTO):
maker_id = batch_users.makerID
for user in batch_users.users:
self._insert_new_user(user)
code = self._create_code(user, maker_id)
await self._insert_new_user(user)
await self._create_code(user, maker_id)
if user.groupName and len(user.groupName.strip()) > 0:
self._assign_user_to_group_by_name(user, maker_id)
await self._assign_user_to_group_by_name(user, maker_id)
def _insert_new_user(self, user: UserDTO):
async def _insert_new_user(self, user: UserDTO):
new_user = {
**user.dict(exclude={
'passport_id', 'groupName', 'expiryDate',
'corporate', 'passwordHash', 'passwordSalt'
}),
'id': str(user.id),
'bio': "",
'focus': "academic",
'status': "active",
@@ -155,11 +142,11 @@ class UserService(IUserService):
'subscriptionExpirationDate': user.expiryDate,
'entities': user.entities
}
self._db.users.insert_one(new_user)
await self._db.save_to_db("users", new_user, str(user.id))
def _create_code(self, user: UserDTO, maker_id: str) -> str:
async def _create_code(self, user: UserDTO, maker_id: str) -> str:
code = shortuuid.ShortUUID().random(length=6)
self._db.codes.insert_one({
await self._db.save_to_db("codes", {
'id': code,
'code': code,
'creator': maker_id,
@@ -170,34 +157,32 @@ class UserService(IUserService):
'email': user.email,
'name': user.name,
'passport_id': user.passport_id
})
}, code)
return code
def _assign_user_to_group_by_name(self, user: UserDTO, maker_id: str):
async def _assign_user_to_group_by_name(self, user: UserDTO, maker_id: str):
user_id = str(user.id)
groups = list(self._db.groups.find(
{
groups = await self._db.find("groups", {
"admin": maker_id,
"name": user.groupName.strip()
}
))
})
if len(groups) == 0:
new_group = {
'id': str(uuid.uuid4()),
'admin': maker_id,
'name': user.groupName.strip(),
'participants': [user_id],
'disableEditing': False,
}
self._db.groups.insert_one(new_group)
await self._db.save_to_db("groups", new_group, str(uuid.uuid4()))
else:
group = groups[0]
participants = group["participants"]
if user_id not in participants:
participants.append(user_id)
self._db.groups.update_one(
await self._db.update(
"groups",
{"id": group["id"]},
{"$set": {"participants": participants}}
)