devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Batch import wasn't updated

Browse Source
This commit is contained in:
Carlos-Mesquita
2024-11-06 11:01:39 +00:00
parent e51cd891d2
commit a2e96f8e54
18 changed files with 124 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/services/abc/exam/listening.py
View File

@@ -16,6 +16,10 @@ class IListeningService(ABC):
async def get_listening_question(self, section: int, dto):
pass
@abstractmethod
async def generate_mp3(self, dto) -> bytes:
pass
@abstractmethod
async def get_dialog_from_audio(self, upload: UploadFile):
pass

2
app/services/abc/third_parties/tts.py
View File

@@ -9,7 +9,7 @@ class ITextToSpeechService(ABC):
pass
@abstractmethod
async def text_to_speech(self, text: Union[list[str], str], file_name: str):
async def text_to_speech(self, dialog) -> bytes:
pass
@abstractmethod

2
app/services/abc/user.py
View File

@@ -6,5 +6,5 @@ from app.dtos.user_batch import BatchUsersDTO
class IUserService(ABC):
@abstractmethod
async def fetch_tips(self, batch: BatchUsersDTO):
async def batch_users(self, batch: BatchUsersDTO):
pass

5
app/services/impl/exam/listening/__init__.py
View File

@@ -7,7 +7,7 @@ from typing import Dict, List
from starlette.datastructures import UploadFile
from app.dtos.listening import GenerateListeningExercises
from app.dtos.listening import GenerateListeningExercises, Dialog
from app.repositories.abc import IFileStorage, IDocumentStore
from app.services.abc import IListeningService, ILLMService, ITextToSpeechService, ISpeechToTextService
from app.configs.question_templates import getListeningTemplate, getListeningPartTemplate
@@ -135,6 +135,9 @@ class ListeningService(IListeningService):
return {"exercises": exercises}
async def generate_mp3(self, dto: Dialog) -> bytes:
return await self._tts.text_to_speech(dto)
async def save_listening(self, parts: list[dict], min_timer: int, difficulty: str, listening_id: str):
template = getListeningTemplate()
template['difficulty'] = difficulty

33
app/services/impl/third_parties/aws_polly.py
View File

@@ -4,6 +4,7 @@ from typing import Union
import aiofiles
from aiobotocore.client import BaseClient
from app.dtos.listening import Dialog
from app.services.abc import ITextToSpeechService
from app.configs.constants import NeuralVoices
@@ -22,14 +23,15 @@ class AWSPolly(ITextToSpeechService):
)
return await tts_response['AudioStream'].read()
async def text_to_speech(self, text: Union[list[str], str], file_name: str):
if isinstance(text, str):
audio_segments = await self._text_to_speech(text)
elif isinstance(text, list):
audio_segments = await self._conversation_to_speech(text)
else:
async def text_to_speech(self, dialog: Dialog) -> bytes:
if not dialog.conversation and not dialog.monologue:
raise ValueError("Unsupported argument for text_to_speech")
if not dialog.conversation:
audio_segments = await self._text_to_speech(dialog.monologue)
else:
audio_segments = await self._conversation_to_speech(dialog)
final_message = await self.synthesize_speech(
"This audio recording, for the listening exercise, has finished.",
"Stephen"
@@ -40,27 +42,26 @@ class AWSPolly(ITextToSpeechService):
# Combine the audio segments into a single audio file
combined_audio = b"".join(audio_segments)
# Save the combined audio to a single file
async with aiofiles.open(file_name, "wb") as f:
await f.write(combined_audio)
print("Speech segments saved to " + file_name)
return combined_audio
# Save the combined audio to a single file
#async with aiofiles.open(file_name, "wb") as f:
# await f.write(combined_audio)
#print("Speech segments saved to " + file_name)
async def _text_to_speech(self, text: str):
voice = random.choice(NeuralVoices.ALL_NEURAL_VOICES)['Id']
# Initialize an empty list to store audio segments
audio_segments = []
for part in self._divide_text(text):
audio_segments.append(await self.synthesize_speech(part, voice))
return audio_segments
async def _conversation_to_speech(self, conversation: list):
# Initialize an empty list to store audio segments
async def _conversation_to_speech(self, dialog: Dialog):
audio_segments = []
# Iterate through the text segments, convert to audio segments, and store them
for segment in conversation:
audio_segments.append(await self.synthesize_speech(segment["text"], segment["voice"]))
for convo_payload in dialog.conversation:
audio_segments.append(await self.synthesize_speech(convo_payload.text, convo_payload.voice))
return audio_segments

10
app/services/impl/training/training.py
View File

@@ -1,4 +1,5 @@
import re
import uuid
from datetime import datetime
from functools import reduce
from logging import getLogger
@@ -23,9 +24,9 @@ class TrainingService(ITrainingService):
]
# strategy word_link ct_focus reading_skill word_partners writing_skill language_for_writing
def __init__(self, llm: ILLMService, firestore: IDocumentStore, training_kb: IKnowledgeBase):
def __init__(self, llm: ILLMService, document_store: IDocumentStore, training_kb: IKnowledgeBase):
self._llm = llm
self._db = firestore
self._db = document_store
self._kb = training_kb
self._logger = getLogger(__name__)
@@ -96,16 +97,15 @@ class TrainingService(ITrainingService):
for area in training_content.weak_areas:
weak_areas["weak_areas"].append(area.dict())
new_id = str(uuid.uuid4())
training_doc = {
'id': new_id,
'created_at': int(datetime.now().timestamp() * 1000),
**exam_map,
**usefull_tips.dict(),
**weak_areas,
"user": user
}
doc_id = await self._db.save_to_db('training', training_doc)
new_id = await self._db.save_to_db('training', training_doc)
return {
"id": new_id
}

57
app/services/impl/user.py
View File

@@ -2,20 +2,17 @@ import os
import subprocess
import time
import uuid
import pandas as pd
import shortuuid
from datetime import datetime
from logging import getLogger
import pandas as pd
from typing import Dict
import shortuuid
from pymongo.database import Database
from app.dtos.user_batch import BatchUsersDTO, UserDTO
from app.helpers import FileHelper
from app.repositories.abc import IDocumentStore
from app.services.abc import IUserService
@@ -34,14 +31,14 @@ class UserService(IUserService):
"speaking": 0,
}
def __init__(self, mongo: Database):
self._db: Database = mongo
def __init__(self, document_store: IDocumentStore):
self._db = document_store
self._logger = getLogger(__name__)
def fetch_tips(self, batch: BatchUsersDTO):
def batch_users(self, batch_dto: BatchUsersDTO):
file_name = f'{uuid.uuid4()}.csv'
path = f'./tmp/{file_name}'
self._generate_firebase_auth_csv(batch, path)
self._generate_firebase_auth_csv(batch_dto, path)
result = self._upload_users('./tmp', file_name)
if result.returncode != 0:
@@ -49,20 +46,11 @@ class UserService(IUserService):
self._logger.error(error_msg)
return error_msg
self._init_users(batch)
self._init_users(batch_dto)
FileHelper.remove_file(path)
return {"ok": True}
@staticmethod
def _map_to_batch(request_data: Dict) -> BatchUsersDTO:
users_list = [{**user} for user in request_data["users"]]
for user in users_list:
user["studentID"] = str(user["studentID"])
users: list[UserDTO] = [UserDTO(**user) for user in users_list]
return BatchUsersDTO(makerID=request_data["makerID"], users=users)
@staticmethod
def _generate_firebase_auth_csv(batch_dto: BatchUsersDTO, path: str):
# https://firebase.google.com/docs/cli/auth#file_format
@@ -127,22 +115,21 @@ class UserService(IUserService):
result = subprocess.run(command, shell=True, cwd=directory, capture_output=True, text=True)
return result
def _init_users(self, batch_users: BatchUsersDTO):
async def _init_users(self, batch_users: BatchUsersDTO):
maker_id = batch_users.makerID
for user in batch_users.users:
self._insert_new_user(user)
code = self._create_code(user, maker_id)
await self._insert_new_user(user)
await self._create_code(user, maker_id)
if user.groupName and len(user.groupName.strip()) > 0:
self._assign_user_to_group_by_name(user, maker_id)
await self._assign_user_to_group_by_name(user, maker_id)
def _insert_new_user(self, user: UserDTO):
async def _insert_new_user(self, user: UserDTO):
new_user = {
**user.dict(exclude={
'passport_id', 'groupName', 'expiryDate',
'corporate', 'passwordHash', 'passwordSalt'
}),
'id': str(user.id),
'bio': "",
'focus': "academic",
'status': "active",
@@ -155,11 +142,11 @@ class UserService(IUserService):
'subscriptionExpirationDate': user.expiryDate,
'entities': user.entities
}
self._db.users.insert_one(new_user)
await self._db.save_to_db("users", new_user, str(user.id))
def _create_code(self, user: UserDTO, maker_id: str) -> str:
async def _create_code(self, user: UserDTO, maker_id: str) -> str:
code = shortuuid.ShortUUID().random(length=6)
self._db.codes.insert_one({
await self._db.save_to_db("codes", {
'id': code,
'code': code,
'creator': maker_id,
@@ -170,34 +157,32 @@ class UserService(IUserService):
'email': user.email,
'name': user.name,
'passport_id': user.passport_id
})
}, code)
return code
def _assign_user_to_group_by_name(self, user: UserDTO, maker_id: str):
async def _assign_user_to_group_by_name(self, user: UserDTO, maker_id: str):
user_id = str(user.id)
groups = list(self._db.groups.find(
{
groups = await self._db.find("groups", {
"admin": maker_id,
"name": user.groupName.strip()
}
))
})
if len(groups) == 0:
new_group = {
'id': str(uuid.uuid4()),
'admin': maker_id,
'name': user.groupName.strip(),
'participants': [user_id],
'disableEditing': False,
}
self._db.groups.insert_one(new_group)
await self._db.save_to_db("groups", new_group, str(uuid.uuid4()))
else:
group = groups[0]
participants = group["participants"]
if user_id not in participants:
participants.append(user_id)
self._db.groups.update_one(
await self._db.update(
"groups",
{"id": group["id"]},
{"$set": {"participants": participants}}
)