From 69820688644d610217047effd040c08ba42ea007 Mon Sep 17 00:00:00 2001 From: Carlos-Mesquita Date: Tue, 10 Dec 2024 22:24:40 +0000 Subject: [PATCH 1/4] Brushed up the backend, added writing task 1 academic prompt gen and grading ENCOA-274 --- Dockerfile | 2 +- app.py | 2 +- app/api/__init__.py | 28 - app/api/writing.py | 27 - app/controllers/abc/user.py | 10 - app/controllers/abc/writing.py | 8 - app/controllers/impl/writing.py | 11 - app/dtos/__init__.py | 0 app/dtos/exams/__init__.py | 0 app/repositories/__init__.py | 0 app/repositories/impl/__init__.py | 8 - app/services/__init__.py | 0 app/server.py => ielts_be/__init__.py | 312 +++---- ielts_be/api/__init__.py | 15 + ielts_be/api/exam/__init__.py | 16 + {app/api => ielts_be/api/exam}/grade.py | 6 +- {app/api => ielts_be/api/exam}/level.py | 6 +- {app/api => ielts_be/api/exam}/listening.py | 8 +- {app/api => ielts_be/api/exam}/reading.py | 8 +- {app/api => ielts_be/api/exam}/speaking.py | 8 +- ielts_be/api/exam/writing.py | 42 + {app => ielts_be}/api/home.py | 0 {app => ielts_be}/api/training.py | 6 +- {app => ielts_be}/api/user.py | 6 +- {app => ielts_be}/configs/__init__.py | 0 {app => ielts_be}/configs/constants.py | 0 .../configs/dependency_injection.py | 18 +- {app => ielts_be}/configs/logging/__init__.py | 0 {app => ielts_be}/configs/logging/filters.py | 0 .../configs/logging/formatters.py | 0 .../configs/logging/logging_config.json | 4 +- .../configs/logging/queue_handler.py | 6 +- .../configs/question_templates.py | 0 ielts_be/controllers/__init__.py | 3 + ielts_be/controllers/abc/__init__.py | 11 + .../controllers/abc/exam}/__init__.py | 32 +- .../controllers/abc/exam}/level.py | 0 .../controllers/abc/exam}/listening.py | 0 .../controllers/abc/exam}/reading.py | 0 .../controllers/abc/exam}/speaking.py | 3 - ielts_be/controllers/abc/exam/writing.py | 14 + {app => ielts_be}/controllers/abc/grade.py | 2 +- {app => ielts_be}/controllers/abc/training.py | 0 ielts_be/controllers/abc/user.py | 8 + ielts_be/controllers/impl/__init__.py | 12 + .../controllers/impl/exam}/__init__.py | 32 +- .../controllers/impl/exam}/level.py | 6 +- .../controllers/impl/exam}/listening.py | 6 +- .../controllers/impl/exam}/reading.py | 6 +- .../controllers/impl/exam}/speaking.py | 6 +- ielts_be/controllers/impl/exam/writing.py | 19 + {app => ielts_be}/controllers/impl/grade.py | 16 +- .../controllers/impl/training.py | 6 +- {app => ielts_be}/controllers/impl/user.py | 6 +- {app => ielts_be/dtos}/__init__.py | 0 {app => ielts_be}/dtos/evaluation.py | 0 .../dtos/exams}/__init__.py | 0 {app => ielts_be}/dtos/exams/level.py | 0 {app => ielts_be}/dtos/exams/listening.py | 0 {app => ielts_be}/dtos/exams/reading.py | 0 {app => ielts_be}/dtos/level.py | 2 +- {app => ielts_be}/dtos/listening.py | 2 +- {app => ielts_be}/dtos/reading.py | 2 +- {app => ielts_be}/dtos/sheet.py | 0 {app => ielts_be}/dtos/speaking.py | 2 - {app => ielts_be}/dtos/training.py | 0 {app => ielts_be}/dtos/user_batch.py | 0 {app => ielts_be}/dtos/video.py | 1 - {app => ielts_be}/dtos/writing.py | 4 + {app => ielts_be}/exceptions/__init__.py | 0 {app => ielts_be}/exceptions/exceptions.py | 0 {app => ielts_be}/helpers/__init__.py | 0 {app => ielts_be}/helpers/exercises.py | 0 {app => ielts_be}/helpers/file.py | 6 + {app => ielts_be}/helpers/text.py | 0 {app => ielts_be}/helpers/token_counter.py | 0 {app => ielts_be}/mappers/__init__.py | 0 {app => ielts_be}/mappers/level.py | 4 +- {app => ielts_be}/mappers/listening.py | 2 +- {app => ielts_be}/mappers/reading.py | 2 +- {app => ielts_be}/middlewares/__init__.py | 0 .../middlewares/authentication.py | 0 .../middlewares/authorization.py | 2 +- ielts_be/repositories/__init__.py | 3 + .../repositories/abc/__init__.py | 0 .../repositories/abc/document_store.py | 0 .../repositories/abc/file_storage.py | 0 ielts_be/repositories/impl/__init__.py | 6 + .../impl/document_stores/__init__.py | 4 +- .../impl/document_stores/firestore.py | 2 +- .../impl/document_stores/mongo.py | 2 +- .../impl/file_storage/__init__.py | 0 .../impl/file_storage/firebase.py | 2 +- ielts_be/services/__init__.py | 3 + {app => ielts_be}/services/abc/__init__.py | 0 {app => ielts_be}/services/abc/evaluation.py | 3 +- .../services/abc/exam/__init__.py | 0 .../services/abc/exam/exercises.py | 0 {app => ielts_be}/services/abc/exam/grade.py | 0 {app => ielts_be}/services/abc/exam/level.py | 5 - .../services/abc/exam/listening.py | 0 .../services/abc/exam/reading.py | 0 .../services/abc/exam/speaking.py | 0 .../services/abc/exam/writing.py | 10 +- .../services/abc/third_parties/__init__.py | 0 .../services/abc/third_parties/ai_detector.py | 0 .../services/abc/third_parties/llm.py | 0 .../services/abc/third_parties/stt.py | 0 .../services/abc/third_parties/tts.py | 0 .../services/abc/third_parties/vid_gen.py | 4 +- .../services/abc/training/__init__.py | 0 {app => ielts_be}/services/abc/training/kb.py | 0 .../services/abc/training/training.py | 0 {app => ielts_be}/services/abc/user.py | 2 +- {app => ielts_be}/services/impl/__init__.py | 0 .../services/impl/exam/__init__.py | 2 + .../services/impl/exam/evaluation.py | 15 +- {app => ielts_be}/services/impl/exam/grade.py | 4 +- .../services/impl/exam/level/__init__.py | 8 +- .../impl/exam/level/exercises/__init__.py | 2 +- .../impl/exam/level/exercises/blank_space.py | 4 +- .../impl/exam/level/exercises/fill_blanks.py | 4 +- .../exam/level/exercises/multiple_choice.py | 6 +- .../impl/exam/level/exercises/passage_utas.py | 6 +- .../impl/exam/level/full_exams/__init__.py | 0 .../impl/exam/level/full_exams/custom.py | 4 +- .../impl/exam/level/full_exams/level_utas.py | 2 +- .../services/impl/exam/level/mc_variants.json | 0 .../services/impl/exam/level/upload.py | 12 +- .../services/impl/exam/listening/__init__.py | 10 +- .../impl/exam/listening/import_listening.py | 8 +- .../impl/exam/listening/write_blank_forms.py | 6 +- .../impl/exam/listening/write_blank_notes.py | 6 +- .../impl/exam/listening/write_blanks.py | 6 +- .../services/impl/exam/reading/__init__.py | 8 +- .../services/impl/exam/reading/fill_blanks.py | 6 +- .../services/impl/exam/reading/idea_match.py | 6 +- .../impl/exam/reading/import_reading.py | 8 +- .../impl/exam/reading/paragraph_match.py | 6 +- .../impl/exam/reading/write_blanks.py | 6 +- .../services/impl/exam/shared/__init__.py | 0 .../impl/exam/shared/multiple_choice.py | 6 +- .../services/impl/exam/shared/true_false.py | 6 +- .../services/impl/exam/speaking/__init__.py | 168 ++++ .../services/impl/exam/speaking/grade.py | 784 +++++++----------- .../services/impl/exam/writing/__init__.py | 80 ++ .../services/impl/exam/writing/academic.py | 48 ++ .../services/impl/exam/writing/general.py | 44 + .../services/impl/exam/writing/grade.py | 475 +++++------ .../services/impl/third_parties/__init__.py | 0 .../services/impl/third_parties/aws_polly.py | 8 +- .../impl/third_parties/elai/__init__.py | 4 +- .../impl/third_parties/elai/avatars.json | 0 .../impl/third_parties/elai/conf.json | 0 .../services/impl/third_parties/gpt_zero.py | 2 +- .../impl/third_parties/heygen/__init__.py | 9 +- .../impl/third_parties/heygen/avatars.json | 0 .../services/impl/third_parties/openai.py | 6 +- .../services/impl/third_parties/whisper.py | 2 +- .../services/impl/training/__init__.py | 0 .../services/impl/training/kb.py | 2 +- .../services/impl/training/training.py | 13 +- {app => ielts_be}/services/impl/user.py | 8 +- {app => ielts_be}/utils/__init__.py | 0 {app => ielts_be}/utils/handle_exception.py | 0 ielts_be/utils/image_to_b64.py | 10 + {app => ielts_be}/utils/logger.py | 0 167 files changed, 1411 insertions(+), 1229 deletions(-) delete mode 100644 app/api/__init__.py delete mode 100644 app/api/writing.py delete mode 100644 app/controllers/abc/user.py delete mode 100644 app/controllers/abc/writing.py delete mode 100644 app/controllers/impl/writing.py delete mode 100644 app/dtos/__init__.py delete mode 100644 app/dtos/exams/__init__.py delete mode 100644 app/repositories/__init__.py delete mode 100644 app/repositories/impl/__init__.py delete mode 100644 app/services/__init__.py rename app/server.py => ielts_be/__init__.py (89%) create mode 100644 ielts_be/api/__init__.py create mode 100644 ielts_be/api/exam/__init__.py rename {app/api => ielts_be/api/exam}/grade.py (87%) rename {app/api => ielts_be/api/exam}/level.py (87%) rename {app/api => ielts_be/api/exam}/listening.py (84%) rename {app/api => ielts_be/api/exam}/reading.py (83%) rename {app/api => ielts_be/api/exam}/speaking.py (87%) create mode 100644 ielts_be/api/exam/writing.py rename {app => ielts_be}/api/home.py (100%) rename {app => ielts_be}/api/training.py (79%) rename {app => ielts_be}/api/user.py (69%) rename {app => ielts_be}/configs/__init__.py (100%) rename {app => ielts_be}/configs/constants.py (100%) rename {app => ielts_be}/configs/dependency_injection.py (88%) rename {app => ielts_be}/configs/logging/__init__.py (100%) rename {app => ielts_be}/configs/logging/filters.py (100%) rename {app => ielts_be}/configs/logging/formatters.py (100%) rename {app => ielts_be}/configs/logging/logging_config.json (85%) rename {app => ielts_be}/configs/logging/queue_handler.py (89%) rename {app => ielts_be}/configs/question_templates.py (100%) create mode 100644 ielts_be/controllers/__init__.py create mode 100644 ielts_be/controllers/abc/__init__.py rename {app/controllers/abc => ielts_be/controllers/abc/exam}/__init__.py (62%) rename {app/controllers/abc => ielts_be/controllers/abc/exam}/level.py (100%) rename {app/controllers/abc => ielts_be/controllers/abc/exam}/listening.py (100%) rename {app/controllers/abc => ielts_be/controllers/abc/exam}/reading.py (100%) rename {app/controllers/abc => ielts_be/controllers/abc/exam}/speaking.py (83%) create mode 100644 ielts_be/controllers/abc/exam/writing.py rename {app => ielts_be}/controllers/abc/grade.py (90%) rename {app => ielts_be}/controllers/abc/training.py (100%) create mode 100644 ielts_be/controllers/abc/user.py create mode 100644 ielts_be/controllers/impl/__init__.py rename {app/controllers/impl => ielts_be/controllers/impl/exam}/__init__.py (63%) rename {app/controllers/impl => ielts_be/controllers/impl/exam}/level.py (83%) rename {app/controllers/impl => ielts_be/controllers/impl/exam}/listening.py (85%) rename {app/controllers/impl => ielts_be/controllers/impl/exam}/reading.py (83%) rename {app/controllers/impl => ielts_be/controllers/impl/exam}/speaking.py (80%) create mode 100644 ielts_be/controllers/impl/exam/writing.py rename {app => ielts_be}/controllers/impl/grade.py (86%) rename {app => ielts_be}/controllers/impl/training.py (73%) rename {app => ielts_be}/controllers/impl/user.py (60%) rename {app => ielts_be/dtos}/__init__.py (100%) rename {app => ielts_be}/dtos/evaluation.py (100%) rename {app/controllers => ielts_be/dtos/exams}/__init__.py (100%) rename {app => ielts_be}/dtos/exams/level.py (100%) rename {app => ielts_be}/dtos/exams/listening.py (100%) rename {app => ielts_be}/dtos/exams/reading.py (100%) rename {app => ielts_be}/dtos/level.py (87%) rename {app => ielts_be}/dtos/listening.py (86%) rename {app => ielts_be}/dtos/reading.py (84%) rename {app => ielts_be}/dtos/sheet.py (100%) rename {app => ielts_be}/dtos/speaking.py (81%) rename {app => ielts_be}/dtos/training.py (100%) rename {app => ielts_be}/dtos/user_batch.py (100%) rename {app => ielts_be}/dtos/video.py (95%) rename {app => ielts_be}/dtos/writing.py (65%) rename {app => ielts_be}/exceptions/__init__.py (100%) rename {app => ielts_be}/exceptions/exceptions.py (100%) rename {app => ielts_be}/helpers/__init__.py (100%) rename {app => ielts_be}/helpers/exercises.py (100%) rename {app => ielts_be}/helpers/file.py (92%) rename {app => ielts_be}/helpers/text.py (100%) rename {app => ielts_be}/helpers/token_counter.py (100%) rename {app => ielts_be}/mappers/__init__.py (100%) rename {app => ielts_be}/mappers/level.py (92%) rename {app => ielts_be}/mappers/listening.py (98%) rename {app => ielts_be}/mappers/reading.py (97%) rename {app => ielts_be}/middlewares/__init__.py (100%) rename {app => ielts_be}/middlewares/authentication.py (100%) rename {app => ielts_be}/middlewares/authorization.py (90%) create mode 100644 ielts_be/repositories/__init__.py rename {app => ielts_be}/repositories/abc/__init__.py (100%) rename {app => ielts_be}/repositories/abc/document_store.py (100%) rename {app => ielts_be}/repositories/abc/file_storage.py (100%) create mode 100644 ielts_be/repositories/impl/__init__.py rename {app => ielts_be}/repositories/impl/document_stores/__init__.py (56%) rename {app => ielts_be}/repositories/impl/document_stores/firestore.py (95%) rename {app => ielts_be}/repositories/impl/document_stores/mongo.py (93%) rename {app => ielts_be}/repositories/impl/file_storage/__init__.py (100%) rename {app => ielts_be}/repositories/impl/file_storage/firebase.py (96%) create mode 100644 ielts_be/services/__init__.py rename {app => ielts_be}/services/abc/__init__.py (100%) rename {app => ielts_be}/services/abc/evaluation.py (87%) rename {app => ielts_be}/services/abc/exam/__init__.py (100%) rename {app => ielts_be}/services/abc/exam/exercises.py (100%) rename {app => ielts_be}/services/abc/exam/grade.py (100%) rename {app => ielts_be}/services/abc/exam/level.py (90%) rename {app => ielts_be}/services/abc/exam/listening.py (100%) rename {app => ielts_be}/services/abc/exam/reading.py (100%) rename {app => ielts_be}/services/abc/exam/speaking.py (100%) rename {app => ielts_be}/services/abc/exam/writing.py (52%) rename {app => ielts_be}/services/abc/third_parties/__init__.py (100%) rename {app => ielts_be}/services/abc/third_parties/ai_detector.py (100%) rename {app => ielts_be}/services/abc/third_parties/llm.py (100%) rename {app => ielts_be}/services/abc/third_parties/stt.py (100%) rename {app => ielts_be}/services/abc/third_parties/tts.py (100%) rename {app => ielts_be}/services/abc/third_parties/vid_gen.py (78%) rename {app => ielts_be}/services/abc/training/__init__.py (100%) rename {app => ielts_be}/services/abc/training/kb.py (100%) rename {app => ielts_be}/services/abc/training/training.py (100%) rename {app => ielts_be}/services/abc/user.py (71%) rename {app => ielts_be}/services/impl/__init__.py (100%) rename {app => ielts_be}/services/impl/exam/__init__.py (81%) rename {app => ielts_be}/services/impl/exam/evaluation.py (88%) rename {app => ielts_be}/services/impl/exam/grade.py (96%) rename {app => ielts_be}/services/impl/exam/level/__init__.py (95%) rename {app => ielts_be}/services/impl/exam/level/exercises/__init__.py (85%) rename {app => ielts_be}/services/impl/exam/level/exercises/blank_space.py (92%) rename app/services/impl/exam/level/exercises/fillBlanks.py => ielts_be/services/impl/exam/level/exercises/fill_blanks.py (94%) rename {app => ielts_be}/services/impl/exam/level/exercises/multiple_choice.py (95%) rename {app => ielts_be}/services/impl/exam/level/exercises/passage_utas.py (94%) rename {app => ielts_be}/services/impl/exam/level/full_exams/__init__.py (100%) rename {app => ielts_be}/services/impl/exam/level/full_exams/custom.py (99%) rename {app => ielts_be}/services/impl/exam/level/full_exams/level_utas.py (98%) rename {app => ielts_be}/services/impl/exam/level/mc_variants.json (100%) rename {app => ielts_be}/services/impl/exam/level/upload.py (96%) rename {app => ielts_be}/services/impl/exam/listening/__init__.py (97%) rename {app => ielts_be}/services/impl/exam/listening/import_listening.py (97%) rename {app => ielts_be}/services/impl/exam/listening/write_blank_forms.py (91%) rename {app => ielts_be}/services/impl/exam/listening/write_blank_notes.py (93%) rename {app => ielts_be}/services/impl/exam/listening/write_blanks.py (90%) rename {app => ielts_be}/services/impl/exam/reading/__init__.py (96%) rename {app => ielts_be}/services/impl/exam/reading/fill_blanks.py (93%) rename {app => ielts_be}/services/impl/exam/reading/idea_match.py (90%) rename {app => ielts_be}/services/impl/exam/reading/import_reading.py (98%) rename {app => ielts_be}/services/impl/exam/reading/paragraph_match.py (92%) rename {app => ielts_be}/services/impl/exam/reading/write_blanks.py (90%) rename {app => ielts_be}/services/impl/exam/shared/__init__.py (100%) rename {app => ielts_be}/services/impl/exam/shared/multiple_choice.py (91%) rename {app => ielts_be}/services/impl/exam/shared/true_false.py (92%) create mode 100644 ielts_be/services/impl/exam/speaking/__init__.py rename app/services/impl/exam/speaking.py => ielts_be/services/impl/exam/speaking/grade.py (60%) create mode 100644 ielts_be/services/impl/exam/writing/__init__.py create mode 100644 ielts_be/services/impl/exam/writing/academic.py create mode 100644 ielts_be/services/impl/exam/writing/general.py rename app/services/impl/exam/writing.py => ielts_be/services/impl/exam/writing/grade.py (62%) rename {app => ielts_be}/services/impl/third_parties/__init__.py (100%) rename {app => ielts_be}/services/impl/third_parties/aws_polly.py (91%) rename {app => ielts_be}/services/impl/third_parties/elai/__init__.py (96%) rename {app => ielts_be}/services/impl/third_parties/elai/avatars.json (100%) rename {app => ielts_be}/services/impl/third_parties/elai/conf.json (100%) rename {app => ielts_be}/services/impl/third_parties/gpt_zero.py (92%) rename {app => ielts_be}/services/impl/third_parties/heygen/__init__.py (94%) rename {app => ielts_be}/services/impl/third_parties/heygen/avatars.json (100%) rename {app => ielts_be}/services/impl/third_parties/openai.py (94%) rename {app => ielts_be}/services/impl/third_parties/whisper.py (95%) rename {app => ielts_be}/services/impl/training/__init__.py (100%) rename {app => ielts_be}/services/impl/training/kb.py (96%) rename {app => ielts_be}/services/impl/training/training.py (96%) rename {app => ielts_be}/services/impl/user.py (94%) rename {app => ielts_be}/utils/__init__.py (100%) rename {app => ielts_be}/utils/handle_exception.py (100%) create mode 100644 ielts_be/utils/image_to_b64.py rename {app => ielts_be}/utils/logger.py (100%) diff --git a/Dockerfile b/Dockerfile index 6afb7e6..97e2f7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,4 +38,4 @@ EXPOSE 8000 # For environments with multiple CPU cores, increase the number of workers # to be equal to the cores available. # Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. -ENTRYPOINT ["gunicorn", "--bind", "0.0.0.0:8000", "--workers", "1", "--threads", "8", "--timeout", "0", "-k", "uvicorn.workers.UvicornWorker", "app.server:app"] +ENTRYPOINT ["gunicorn", "--bind", "0.0.0.0:8000", "--workers", "1", "--threads", "8", "--timeout", "0", "-k", "uvicorn.workers.UvicornWorker", "ielts_be:app"] diff --git a/app.py b/app.py index df62e97..a757090 100644 --- a/app.py +++ b/app.py @@ -13,7 +13,7 @@ load_dotenv() ) def main(env: str): uvicorn.run( - app="app.server:app", + app="ielts_be:app", host="localhost", port=8000, reload=True if env != "production" else False, diff --git a/app/api/__init__.py b/app/api/__init__.py deleted file mode 100644 index 2c98dd4..0000000 --- a/app/api/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -from fastapi import APIRouter - -from .listening import listening_router -from .reading import reading_router -from .speaking import speaking_router -from .training import training_router -from .writing import writing_router -from .grade import grade_router -from .user import user_router -from .level import level_router - -router = APIRouter(prefix="/api", tags=["Home"]) - -@router.get('/healthcheck') -async def healthcheck(): - return {"healthy": True} - -exercises_router = APIRouter() -exercises_router.include_router(listening_router, prefix="/listening", tags=["Listening"]) -exercises_router.include_router(reading_router, prefix="/reading", tags=["Reading"]) -exercises_router.include_router(speaking_router, prefix="/speaking", tags=["Speaking"]) -exercises_router.include_router(writing_router, prefix="/writing", tags=["Writing"]) -exercises_router.include_router(level_router, prefix="/level", tags=["Level"]) - -router.include_router(grade_router, prefix="/grade", tags=["Grade"]) -router.include_router(training_router, prefix="/training", tags=["Training"]) -router.include_router(user_router, prefix="/user", tags=["Users"]) -router.include_router(exercises_router) diff --git a/app/api/writing.py b/app/api/writing.py deleted file mode 100644 index ed72941..0000000 --- a/app/api/writing.py +++ /dev/null @@ -1,27 +0,0 @@ -import random - -from dependency_injector.wiring import inject, Provide -from fastapi import APIRouter, Path, Query, Depends - -from app.middlewares import Authorized, IsAuthenticatedViaBearerToken -from app.configs.constants import EducationalContent -from app.controllers.abc import IWritingController - -controller = "writing_controller" -writing_router = APIRouter() - - -@writing_router.get( - '/{task}', - dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] -) -@inject -async def generate_writing( - task: int = Path(..., ge=1, le=2), - difficulty: str = Query(default=None), - topic: str = Query(default=None), - writing_controller: IWritingController = Depends(Provide[controller]) -): - difficulty = random.choice(EducationalContent.DIFFICULTIES) if not difficulty else difficulty - topic = random.choice(EducationalContent.MTI_TOPICS) if not topic else topic - return await writing_controller.get_writing_task_general_question(task, topic, difficulty) diff --git a/app/controllers/abc/user.py b/app/controllers/abc/user.py deleted file mode 100644 index c99c3df..0000000 --- a/app/controllers/abc/user.py +++ /dev/null @@ -1,10 +0,0 @@ -from abc import ABC, abstractmethod - -from app.dtos.user_batch import BatchUsersDTO - - -class IUserController(ABC): - - @abstractmethod - async def batch_import(self, batch: BatchUsersDTO): - pass diff --git a/app/controllers/abc/writing.py b/app/controllers/abc/writing.py deleted file mode 100644 index ebb298e..0000000 --- a/app/controllers/abc/writing.py +++ /dev/null @@ -1,8 +0,0 @@ -from abc import ABC, abstractmethod - - -class IWritingController(ABC): - - @abstractmethod - async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): - pass diff --git a/app/controllers/impl/writing.py b/app/controllers/impl/writing.py deleted file mode 100644 index c097d5c..0000000 --- a/app/controllers/impl/writing.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.controllers.abc import IWritingController -from app.services.abc import IWritingService - - -class WritingController(IWritingController): - - def __init__(self, writing_service: IWritingService): - self._service = writing_service - - async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): - return await self._service.get_writing_task_general_question(task, topic, difficulty) diff --git a/app/dtos/__init__.py b/app/dtos/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/dtos/exams/__init__.py b/app/dtos/exams/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/repositories/impl/__init__.py b/app/repositories/impl/__init__.py deleted file mode 100644 index 5415ab4..0000000 --- a/app/repositories/impl/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .document_stores import * -from app.repositories.impl.file_storage.firebase import FirebaseStorage - -__all__ = [ - "FirebaseStorage" -] - -__all__.extend(document_stores.__all__) diff --git a/app/services/__init__.py b/app/services/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/server.py b/ielts_be/__init__.py similarity index 89% rename from app/server.py rename to ielts_be/__init__.py index 2b4ca19..4e25d95 100644 --- a/app/server.py +++ b/ielts_be/__init__.py @@ -1,156 +1,156 @@ -import json -import os -import pathlib -import logging.config -import logging.handlers - -import aioboto3 -import contextlib -from contextlib import asynccontextmanager -from collections import defaultdict -from typing import List -from http import HTTPStatus - -import httpx -from fastapi import FastAPI, Request -from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import RequestValidationError -from fastapi.middleware import Middleware -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse - -import nltk -from starlette import status - -from app.api import router -from app.configs import DependencyInjector -from app.exceptions import CustomException -from app.middlewares import AuthenticationMiddleware, AuthBackend -from app.services.impl import OpenAIWhisper - - -@asynccontextmanager -async def lifespan(_app: FastAPI): - """ - Startup and Shutdown logic is in this lifespan method - - https://fastapi.tiangolo.com/advanced/events/ - """ - - # NLTK required datasets download - nltk.download('words') - nltk.download("punkt") - - # AWS Polly client instantiation - context_stack = contextlib.AsyncExitStack() - session = aioboto3.Session() - polly_client = await context_stack.enter_async_context( - session.client( - 'polly', - region_name='eu-west-1', - aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID") - ) - ) - - http_client = httpx.AsyncClient() - stt = OpenAIWhisper() - - DependencyInjector( - polly_client, - http_client, - stt - ).inject() - - # Setup logging - config_file = pathlib.Path("./app/configs/logging/logging_config.json") - with open(config_file) as f_in: - config = json.load(f_in) - - logging.config.dictConfig(config) - - yield - - stt.close() - await http_client.aclose() - await polly_client.close() - await context_stack.aclose() - - -def setup_listeners(_app: FastAPI) -> None: - @_app.exception_handler(RequestValidationError) - async def custom_form_validation_error(request, exc): - """ - Don't delete request param - """ - reformatted_message = defaultdict(list) - for pydantic_error in exc.errors(): - loc, msg = pydantic_error["loc"], pydantic_error["msg"] - filtered_loc = loc[1:] if loc[0] in ("body", "query", "path") else loc - field_string = ".".join(filtered_loc) - if field_string == "cookie.refresh_token": - return JSONResponse( - status_code=401, - content={"error_code": 401, "message": HTTPStatus.UNAUTHORIZED.description}, - ) - reformatted_message[field_string].append(msg) - - return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, - content=jsonable_encoder( - {"details": "Invalid request!", "errors": reformatted_message} - ), - ) - - @_app.exception_handler(CustomException) - async def custom_exception_handler(request: Request, exc: CustomException): - """ - Don't delete request param - """ - return JSONResponse( - status_code=exc.code, - content={"error_code": exc.error_code, "message": exc.message}, - ) - - @_app.exception_handler(Exception) - async def default_exception_handler(request: Request, exc: Exception): - """ - Don't delete request param - """ - return JSONResponse( - status_code=500, - content=str(exc), - ) - - -def setup_middleware() -> List[Middleware]: - middleware = [ - Middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ), - Middleware( - AuthenticationMiddleware, - backend=AuthBackend() - ) - ] - return middleware - - -def create_app() -> FastAPI: - env = os.getenv("ENV") - _app = FastAPI( - docs_url="/docs" if env != "production" else None, - redoc_url="/redoc" if env != "production" else None, - middleware=setup_middleware(), - lifespan=lifespan - ) - _app.include_router(router) - setup_listeners(_app) - return _app - - -app = create_app() +import json +import os +import pathlib +import logging.config +import logging.handlers + +import aioboto3 +import contextlib +from contextlib import asynccontextmanager +from collections import defaultdict +from typing import List +from http import HTTPStatus + +import httpx +from fastapi import FastAPI, Request +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.middleware import Middleware +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse + +import nltk +from starlette import status + +from ielts_be.api import router +from ielts_be.configs import DependencyInjector +from ielts_be.exceptions import CustomException +from ielts_be.middlewares import AuthenticationMiddleware, AuthBackend +from ielts_be.services.impl import OpenAIWhisper + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + """ + Startup and Shutdown logic is in this lifespan method + + https://fastapi.tiangolo.com/advanced/events/ + """ + + # NLTK required datasets download + nltk.download('words') + nltk.download("punkt") + + # AWS Polly client instantiation + context_stack = contextlib.AsyncExitStack() + session = aioboto3.Session() + polly_client = await context_stack.enter_async_context( + session.client( + 'polly', + region_name='eu-west-1', + aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID") + ) + ) + + http_client = httpx.AsyncClient() + stt = OpenAIWhisper() + + DependencyInjector( + polly_client, + http_client, + stt + ).inject() + + # Setup logging + config_file = pathlib.Path("./ielts_be/configs/logging/logging_config.json") + with open(config_file) as f_in: + config = json.load(f_in) + + logging.config.dictConfig(config) + + yield + + stt.close() + await http_client.aclose() + await polly_client.close() + await context_stack.aclose() + + +def setup_listeners(_app: FastAPI) -> None: + @_app.exception_handler(RequestValidationError) + async def custom_form_validation_error(request, exc): + """ + Don't delete request param + """ + reformatted_message = defaultdict(list) + for pydantic_error in exc.errors(): + loc, msg = pydantic_error["loc"], pydantic_error["msg"] + filtered_loc = loc[1:] if loc[0] in ("body", "query", "path") else loc + field_string = ".".join(filtered_loc) + if field_string == "cookie.refresh_token": + return JSONResponse( + status_code=401, + content={"error_code": 401, "message": HTTPStatus.UNAUTHORIZED.description}, + ) + reformatted_message[field_string].append(msg) + + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content=jsonable_encoder( + {"details": "Invalid request!", "errors": reformatted_message} + ), + ) + + @_app.exception_handler(CustomException) + async def custom_exception_handler(request: Request, exc: CustomException): + """ + Don't delete request param + """ + return JSONResponse( + status_code=exc.code, + content={"error_code": exc.error_code, "message": exc.message}, + ) + + @_app.exception_handler(Exception) + async def default_exception_handler(request: Request, exc: Exception): + """ + Don't delete request param + """ + return JSONResponse( + status_code=500, + content=str(exc), + ) + + +def setup_middleware() -> List[Middleware]: + middleware = [ + Middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ), + Middleware( + AuthenticationMiddleware, + backend=AuthBackend() + ) + ] + return middleware + + +def create_app() -> FastAPI: + env = os.getenv("ENV") + _app = FastAPI( + docs_url="/docs" if env != "production" else None, + redoc_url="/redoc" if env != "production" else None, + middleware=setup_middleware(), + lifespan=lifespan + ) + _app.include_router(router) + setup_listeners(_app) + return _app + + +app = create_app() diff --git a/ielts_be/api/__init__.py b/ielts_be/api/__init__.py new file mode 100644 index 0000000..1b70727 --- /dev/null +++ b/ielts_be/api/__init__.py @@ -0,0 +1,15 @@ +from fastapi import APIRouter + +from .training import training_router +from .user import user_router +from .exam import exam_router + +router = APIRouter(prefix="/api", tags=["Home"]) + +@router.get('/healthcheck') +async def healthcheck(): + return {"healthy": True} + +router.include_router(training_router, prefix="/training", tags=["Training"]) +router.include_router(user_router, prefix="/user", tags=["Users"]) +router.include_router(exam_router) diff --git a/ielts_be/api/exam/__init__.py b/ielts_be/api/exam/__init__.py new file mode 100644 index 0000000..79490b3 --- /dev/null +++ b/ielts_be/api/exam/__init__.py @@ -0,0 +1,16 @@ +from fastapi import APIRouter + +from .listening import listening_router +from .reading import reading_router +from .speaking import speaking_router +from .writing import writing_router +from .level import level_router +from .grade import grade_router + +exam_router = APIRouter() +exam_router.include_router(listening_router, prefix="/listening", tags=["Listening"]) +exam_router.include_router(reading_router, prefix="/reading", tags=["Reading"]) +exam_router.include_router(speaking_router, prefix="/speaking", tags=["Speaking"]) +exam_router.include_router(writing_router, prefix="/writing", tags=["Writing"]) +exam_router.include_router(level_router, prefix="/level", tags=["Level"]) +exam_router.include_router(grade_router, prefix="/grade", tags=["Grade"]) diff --git a/app/api/grade.py b/ielts_be/api/exam/grade.py similarity index 87% rename from app/api/grade.py rename to ielts_be/api/exam/grade.py index 268142b..2428610 100644 --- a/app/api/grade.py +++ b/ielts_be/api/exam/grade.py @@ -1,9 +1,9 @@ from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Depends, Path, Request, BackgroundTasks -from app.controllers.abc import IGradeController -from app.dtos.writing import WritingGradeTaskDTO -from app.middlewares import Authorized, IsAuthenticatedViaBearerToken +from ielts_be.controllers import IGradeController +from ielts_be.dtos.writing import WritingGradeTaskDTO +from ielts_be.middlewares import Authorized, IsAuthenticatedViaBearerToken controller = "grade_controller" grade_router = APIRouter() diff --git a/app/api/level.py b/ielts_be/api/exam/level.py similarity index 87% rename from app/api/level.py rename to ielts_be/api/exam/level.py index d46049c..1f43f7e 100644 --- a/app/api/level.py +++ b/ielts_be/api/exam/level.py @@ -1,9 +1,9 @@ from dependency_injector.wiring import Provide, inject from fastapi import APIRouter, Depends, UploadFile, Request -from app.dtos.level import LevelExercisesDTO -from app.middlewares import Authorized, IsAuthenticatedViaBearerToken -from app.controllers.abc import ILevelController +from ielts_be.dtos.level import LevelExercisesDTO +from ielts_be.middlewares import Authorized, IsAuthenticatedViaBearerToken +from ielts_be.controllers import ILevelController controller = "level_controller" level_router = APIRouter() diff --git a/app/api/listening.py b/ielts_be/api/exam/listening.py similarity index 84% rename from app/api/listening.py rename to ielts_be/api/exam/listening.py index 070b73c..102913a 100644 --- a/app/api/listening.py +++ b/ielts_be/api/exam/listening.py @@ -3,10 +3,10 @@ import random from dependency_injector.wiring import Provide, inject from fastapi import APIRouter, Depends, Path, Query, UploadFile -from app.middlewares import Authorized, IsAuthenticatedViaBearerToken -from app.controllers.abc import IListeningController -from app.configs.constants import EducationalContent, ListeningExerciseType -from app.dtos.listening import SaveListeningDTO, GenerateListeningExercises, Dialog +from ielts_be.middlewares import Authorized, IsAuthenticatedViaBearerToken +from ielts_be.controllers import IListeningController +from ielts_be.configs.constants import EducationalContent +from ielts_be.dtos.listening import GenerateListeningExercises, Dialog controller = "listening_controller" listening_router = APIRouter() diff --git a/app/api/reading.py b/ielts_be/api/exam/reading.py similarity index 83% rename from app/api/reading.py rename to ielts_be/api/exam/reading.py index 2f998d3..30578fd 100644 --- a/app/api/reading.py +++ b/ielts_be/api/exam/reading.py @@ -4,10 +4,10 @@ from typing import Optional from dependency_injector.wiring import Provide, inject from fastapi import APIRouter, Depends, Path, Query, UploadFile -from app.configs.constants import EducationalContent -from app.dtos.reading import ReadingDTO -from app.middlewares import Authorized, IsAuthenticatedViaBearerToken -from app.controllers.abc import IReadingController +from ielts_be.configs.constants import EducationalContent +from ielts_be.dtos.reading import ReadingDTO +from ielts_be.middlewares import Authorized, IsAuthenticatedViaBearerToken +from ielts_be.controllers import IReadingController controller = "reading_controller" reading_router = APIRouter() diff --git a/app/api/speaking.py b/ielts_be/api/exam/speaking.py similarity index 87% rename from app/api/speaking.py rename to ielts_be/api/exam/speaking.py index e4245fb..4a18a0b 100644 --- a/app/api/speaking.py +++ b/ielts_be/api/exam/speaking.py @@ -4,10 +4,10 @@ from typing import Optional from dependency_injector.wiring import inject, Provide from fastapi import APIRouter, Path, Query, Depends -from app.dtos.speaking import Video -from app.middlewares import Authorized, IsAuthenticatedViaBearerToken -from app.configs.constants import EducationalContent -from app.controllers.abc import ISpeakingController +from ielts_be.dtos.speaking import Video +from ielts_be.middlewares import Authorized, IsAuthenticatedViaBearerToken +from ielts_be.configs.constants import EducationalContent +from ielts_be.controllers import ISpeakingController controller = "speaking_controller" speaking_router = APIRouter() diff --git a/ielts_be/api/exam/writing.py b/ielts_be/api/exam/writing.py new file mode 100644 index 0000000..ed304dd --- /dev/null +++ b/ielts_be/api/exam/writing.py @@ -0,0 +1,42 @@ +import random + +from dependency_injector.wiring import inject, Provide +from fastapi import APIRouter, Path, Query, Depends, UploadFile, File + +from ielts_be.middlewares import Authorized, IsAuthenticatedViaBearerToken +from ielts_be.configs.constants import EducationalContent +from ielts_be.controllers import IWritingController + +controller = "writing_controller" +writing_router = APIRouter() + + +@writing_router.post( + '/{task}/attachment', + dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] +) +@inject +async def generate_writing_academic( + task: int = Path(..., ge=1, le=2), + file: UploadFile = File(...), + difficulty: str = Query(default=None), + writing_controller: IWritingController = Depends(Provide[controller]) +): + difficulty = random.choice(EducationalContent.DIFFICULTIES) if not difficulty else difficulty + return await writing_controller.get_writing_task_academic_question(task, file, difficulty) + + +@writing_router.get( + '/{task}', + dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] +) +@inject +async def generate_writing( + task: int = Path(..., ge=1, le=2), + difficulty: str = Query(default=None), + topic: str = Query(default=None), + writing_controller: IWritingController = Depends(Provide[controller]) +): + difficulty = random.choice(EducationalContent.DIFFICULTIES) if not difficulty else difficulty + topic = random.choice(EducationalContent.MTI_TOPICS) if not topic else topic + return await writing_controller.get_writing_task_general_question(task, topic, difficulty) diff --git a/app/api/home.py b/ielts_be/api/home.py similarity index 100% rename from app/api/home.py rename to ielts_be/api/home.py diff --git a/app/api/training.py b/ielts_be/api/training.py similarity index 79% rename from app/api/training.py rename to ielts_be/api/training.py index 739876e..a4148d0 100644 --- a/app/api/training.py +++ b/ielts_be/api/training.py @@ -1,9 +1,9 @@ from dependency_injector.wiring import Provide, inject from fastapi import APIRouter, Depends, Request -from app.dtos.training import FetchTipsDTO -from app.middlewares import Authorized, IsAuthenticatedViaBearerToken -from app.controllers.abc import ITrainingController +from ielts_be.dtos.training import FetchTipsDTO +from ielts_be.middlewares import Authorized, IsAuthenticatedViaBearerToken +from ielts_be.controllers import ITrainingController controller = "training_controller" training_router = APIRouter() diff --git a/app/api/user.py b/ielts_be/api/user.py similarity index 69% rename from app/api/user.py rename to ielts_be/api/user.py index 2275680..2a3bc0b 100644 --- a/app/api/user.py +++ b/ielts_be/api/user.py @@ -1,9 +1,9 @@ from dependency_injector.wiring import Provide, inject from fastapi import APIRouter, Depends -from app.dtos.user_batch import BatchUsersDTO -from app.middlewares import Authorized, IsAuthenticatedViaBearerToken -from app.controllers.abc import IUserController +from ielts_be.dtos.user_batch import BatchUsersDTO +from ielts_be.middlewares import Authorized, IsAuthenticatedViaBearerToken +from ielts_be.controllers import IUserController controller = "user_controller" user_router = APIRouter() diff --git a/app/configs/__init__.py b/ielts_be/configs/__init__.py similarity index 100% rename from app/configs/__init__.py rename to ielts_be/configs/__init__.py diff --git a/app/configs/constants.py b/ielts_be/configs/constants.py similarity index 100% rename from app/configs/constants.py rename to ielts_be/configs/constants.py diff --git a/app/configs/dependency_injection.py b/ielts_be/configs/dependency_injection.py similarity index 88% rename from app/configs/dependency_injection.py rename to ielts_be/configs/dependency_injection.py index a6e934c..084ffe5 100644 --- a/app/configs/dependency_injection.py +++ b/ielts_be/configs/dependency_injection.py @@ -9,11 +9,9 @@ from httpx import AsyncClient as HTTPClient from dotenv import load_dotenv from sentence_transformers import SentenceTransformer -from app.repositories.impl import * -from app.repositories.impl.document_stores.mongo import MongoDB -from app.services.impl import * -from app.controllers.impl import * -from app.services.impl.exam.evaluation import EvaluationService +from ielts_be.repositories.impl import * +from ielts_be.services.impl import * +from ielts_be.controllers.impl import * load_dotenv() @@ -33,7 +31,7 @@ class DependencyInjector: self._setup_services() self._setup_controllers() self._container.wire( - packages=["app"] + packages=["ielts_be"] ) return self @@ -48,14 +46,14 @@ class DependencyInjector: self._container.tts = providers.Factory(AWSPolly, client=self._container.polly_client) """ - with open('app/services/impl/third_parties/elai/conf.json', 'r') as file: + with open('ielts_be/services/impl/third_parties/elai/conf.json', 'r') as file: elai_conf = json.load(file) - with open('app/services/impl/third_parties/elai/avatars.json', 'r') as file: + with open('ielts_be/services/impl/third_parties/elai/avatars.json', 'r') as file: elai_avatars = json.load(file) """ - with open('app/services/impl/third_parties/heygen/avatars.json', 'r') as file: + with open('ielts_be/services/impl/third_parties/heygen/avatars.json', 'r') as file: heygen_avatars = json.load(file) self._container.vid_gen = providers.Factory( @@ -99,7 +97,7 @@ class DependencyInjector: WritingService, llm=self._container.llm, ai_detector=self._container.ai_detector ) - with open('app/services/impl/exam/level/mc_variants.json', 'r') as file: + with open('ielts_be/services/impl/exam/level/mc_variants.json', 'r') as file: mc_variants = json.load(file) self._container.level_service = providers.Factory( diff --git a/app/configs/logging/__init__.py b/ielts_be/configs/logging/__init__.py similarity index 100% rename from app/configs/logging/__init__.py rename to ielts_be/configs/logging/__init__.py diff --git a/app/configs/logging/filters.py b/ielts_be/configs/logging/filters.py similarity index 100% rename from app/configs/logging/filters.py rename to ielts_be/configs/logging/filters.py diff --git a/app/configs/logging/formatters.py b/ielts_be/configs/logging/formatters.py similarity index 100% rename from app/configs/logging/formatters.py rename to ielts_be/configs/logging/formatters.py diff --git a/app/configs/logging/logging_config.json b/ielts_be/configs/logging/logging_config.json similarity index 85% rename from app/configs/logging/logging_config.json rename to ielts_be/configs/logging/logging_config.json index 61ea4f1..dd56c91 100644 --- a/app/configs/logging/logging_config.json +++ b/ielts_be/configs/logging/logging_config.json @@ -15,7 +15,7 @@ }, "filters": { "error_and_above": { - "()": "app.configs.logging.ErrorAndAboveFilter" + "()": "ielts_be.configs.logging.ErrorAndAboveFilter" } }, "handlers": { @@ -33,7 +33,7 @@ "stream": "ext://sys.stderr" }, "queue_handler": { - "class": "app.configs.logging.QueueListenerHandler", + "class": "ielts_be.configs.logging.QueueListenerHandler", "handlers": [ "cfg://handlers.console", "cfg://handlers.error" diff --git a/app/configs/logging/queue_handler.py b/ielts_be/configs/logging/queue_handler.py similarity index 89% rename from app/configs/logging/queue_handler.py rename to ielts_be/configs/logging/queue_handler.py index 4a6dd98..f91b695 100644 --- a/app/configs/logging/queue_handler.py +++ b/ielts_be/configs/logging/queue_handler.py @@ -4,7 +4,7 @@ from queue import Queue import atexit -class QueueHnadlerHelper: +class QueueHandlerHelper: @staticmethod def resolve_handlers(l): @@ -40,9 +40,9 @@ class QueueHnadlerHelper: class QueueListenerHandler(QueueHandler): def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)): - queue = QueueHnadlerHelper.resolve_queue(queue) + queue = QueueHandlerHelper.resolve_queue(queue) super().__init__(queue) - handlers = QueueHnadlerHelper.resolve_handlers(handlers) + handlers = QueueHandlerHelper.resolve_handlers(handlers) self._listener = QueueListener( self.queue, *handlers, diff --git a/app/configs/question_templates.py b/ielts_be/configs/question_templates.py similarity index 100% rename from app/configs/question_templates.py rename to ielts_be/configs/question_templates.py diff --git a/ielts_be/controllers/__init__.py b/ielts_be/controllers/__init__.py new file mode 100644 index 0000000..621885c --- /dev/null +++ b/ielts_be/controllers/__init__.py @@ -0,0 +1,3 @@ +from .abc import * + +__all__ = abc.__all__ diff --git a/ielts_be/controllers/abc/__init__.py b/ielts_be/controllers/abc/__init__.py new file mode 100644 index 0000000..7e6d5c0 --- /dev/null +++ b/ielts_be/controllers/abc/__init__.py @@ -0,0 +1,11 @@ +from .grade import IGradeController +from .training import ITrainingController +from .user import IUserController +from .exam import * + +__all__ = [ + "IGradeController", + "ITrainingController", + "IUserController", +] +__all__.extend(exam.__all__) diff --git a/app/controllers/abc/__init__.py b/ielts_be/controllers/abc/exam/__init__.py similarity index 62% rename from app/controllers/abc/__init__.py rename to ielts_be/controllers/abc/exam/__init__.py index abdee64..1ab1a7e 100644 --- a/app/controllers/abc/__init__.py +++ b/ielts_be/controllers/abc/exam/__init__.py @@ -1,19 +1,13 @@ -from .level import ILevelController -from .listening import IListeningController -from .reading import IReadingController -from .writing import IWritingController -from .speaking import ISpeakingController -from .grade import IGradeController -from .training import ITrainingController -from .user import IUserController - -__all__ = [ - "IListeningController", - "IReadingController", - "IWritingController", - "ISpeakingController", - "ILevelController", - "IGradeController", - "ITrainingController", - "IUserController", -] +from .level import ILevelController +from .listening import IListeningController +from .reading import IReadingController +from .writing import IWritingController +from .speaking import ISpeakingController + +__all__ = [ + "IListeningController", + "IReadingController", + "IWritingController", + "ISpeakingController", + "ILevelController", +] diff --git a/app/controllers/abc/level.py b/ielts_be/controllers/abc/exam/level.py similarity index 100% rename from app/controllers/abc/level.py rename to ielts_be/controllers/abc/exam/level.py diff --git a/app/controllers/abc/listening.py b/ielts_be/controllers/abc/exam/listening.py similarity index 100% rename from app/controllers/abc/listening.py rename to ielts_be/controllers/abc/exam/listening.py diff --git a/app/controllers/abc/reading.py b/ielts_be/controllers/abc/exam/reading.py similarity index 100% rename from app/controllers/abc/reading.py rename to ielts_be/controllers/abc/exam/reading.py diff --git a/app/controllers/abc/speaking.py b/ielts_be/controllers/abc/exam/speaking.py similarity index 83% rename from app/controllers/abc/speaking.py rename to ielts_be/controllers/abc/exam/speaking.py index 941bf86..73c5a38 100644 --- a/app/controllers/abc/speaking.py +++ b/ielts_be/controllers/abc/exam/speaking.py @@ -1,7 +1,4 @@ from abc import ABC, abstractmethod -from typing import Optional - -from fastapi import BackgroundTasks class ISpeakingController(ABC): diff --git a/ielts_be/controllers/abc/exam/writing.py b/ielts_be/controllers/abc/exam/writing.py new file mode 100644 index 0000000..4fc9e0e --- /dev/null +++ b/ielts_be/controllers/abc/exam/writing.py @@ -0,0 +1,14 @@ +from abc import ABC, abstractmethod + +from fastapi.datastructures import UploadFile + + +class IWritingController(ABC): + + @abstractmethod + async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): + pass + + @abstractmethod + async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: str): + pass diff --git a/app/controllers/abc/grade.py b/ielts_be/controllers/abc/grade.py similarity index 90% rename from app/controllers/abc/grade.py rename to ielts_be/controllers/abc/grade.py index f0c883a..f0d461e 100644 --- a/app/controllers/abc/grade.py +++ b/ielts_be/controllers/abc/grade.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Dict, List, Union +from typing import Dict from fastapi import BackgroundTasks from fastapi.datastructures import FormData diff --git a/app/controllers/abc/training.py b/ielts_be/controllers/abc/training.py similarity index 100% rename from app/controllers/abc/training.py rename to ielts_be/controllers/abc/training.py diff --git a/ielts_be/controllers/abc/user.py b/ielts_be/controllers/abc/user.py new file mode 100644 index 0000000..3d37d37 --- /dev/null +++ b/ielts_be/controllers/abc/user.py @@ -0,0 +1,8 @@ +from abc import ABC, abstractmethod + + +class IUserController(ABC): + + @abstractmethod + async def batch_import(self, batch): + pass diff --git a/ielts_be/controllers/impl/__init__.py b/ielts_be/controllers/impl/__init__.py new file mode 100644 index 0000000..865c438 --- /dev/null +++ b/ielts_be/controllers/impl/__init__.py @@ -0,0 +1,12 @@ +from .training import TrainingController +from .grade import GradeController +from .user import UserController +from .exam import * + +__all__ = [ + "TrainingController", + "GradeController", + "UserController" +] + +__all__.extend(exam.__all__) diff --git a/app/controllers/impl/__init__.py b/ielts_be/controllers/impl/exam/__init__.py similarity index 63% rename from app/controllers/impl/__init__.py rename to ielts_be/controllers/impl/exam/__init__.py index 6671490..71c35da 100644 --- a/app/controllers/impl/__init__.py +++ b/ielts_be/controllers/impl/exam/__init__.py @@ -1,19 +1,13 @@ -from .level import LevelController -from .listening import ListeningController -from .reading import ReadingController -from .speaking import SpeakingController -from .writing import WritingController -from .training import TrainingController -from .grade import GradeController -from .user import UserController - -__all__ = [ - "LevelController", - "ListeningController", - "ReadingController", - "SpeakingController", - "WritingController", - "TrainingController", - "GradeController", - "UserController" -] +from .level import LevelController +from .listening import ListeningController +from .reading import ReadingController +from .speaking import SpeakingController +from .writing import WritingController + +__all__ = [ + "LevelController", + "ListeningController", + "ReadingController", + "SpeakingController", + "WritingController", +] diff --git a/app/controllers/impl/level.py b/ielts_be/controllers/impl/exam/level.py similarity index 83% rename from app/controllers/impl/level.py rename to ielts_be/controllers/impl/exam/level.py index 7317ef0..927e51a 100644 --- a/app/controllers/impl/level.py +++ b/ielts_be/controllers/impl/exam/level.py @@ -1,10 +1,8 @@ from fastapi import UploadFile from typing import Dict, Optional -from watchfiles import awatch - -from app.controllers.abc import ILevelController -from app.services.abc import ILevelService +from ielts_be.controllers import ILevelController +from ielts_be.services import ILevelService class LevelController(ILevelController): diff --git a/app/controllers/impl/listening.py b/ielts_be/controllers/impl/exam/listening.py similarity index 85% rename from app/controllers/impl/listening.py rename to ielts_be/controllers/impl/exam/listening.py index e6dc047..dd555dc 100644 --- a/app/controllers/impl/listening.py +++ b/ielts_be/controllers/impl/exam/listening.py @@ -3,9 +3,9 @@ import io from fastapi import UploadFile from starlette.responses import StreamingResponse, Response -from app.controllers.abc import IListeningController -from app.dtos.listening import GenerateListeningExercises, Dialog -from app.services.abc import IListeningService +from ielts_be.controllers import IListeningController +from ielts_be.services import IListeningService +from ielts_be.dtos.listening import GenerateListeningExercises, Dialog class ListeningController(IListeningController): diff --git a/app/controllers/impl/reading.py b/ielts_be/controllers/impl/exam/reading.py similarity index 83% rename from app/controllers/impl/reading.py rename to ielts_be/controllers/impl/exam/reading.py index a35eddf..d338d5e 100644 --- a/app/controllers/impl/reading.py +++ b/ielts_be/controllers/impl/exam/reading.py @@ -3,9 +3,9 @@ from typing import Optional from fastapi import UploadFile, Response -from app.controllers.abc import IReadingController -from app.dtos.reading import ReadingDTO -from app.services.abc import IReadingService +from ielts_be.controllers import IReadingController +from ielts_be.services import IReadingService +from ielts_be.dtos.reading import ReadingDTO class ReadingController(IReadingController): diff --git a/app/controllers/impl/speaking.py b/ielts_be/controllers/impl/exam/speaking.py similarity index 80% rename from app/controllers/impl/speaking.py rename to ielts_be/controllers/impl/exam/speaking.py index 820c88b..0ca895d 100644 --- a/app/controllers/impl/speaking.py +++ b/ielts_be/controllers/impl/exam/speaking.py @@ -1,9 +1,7 @@ import logging -import random -from typing import Optional -from app.controllers.abc import ISpeakingController -from app.services.abc import ISpeakingService, IVideoGeneratorService +from ielts_be.controllers import ISpeakingController +from ielts_be.services import ISpeakingService, IVideoGeneratorService class SpeakingController(ISpeakingController): diff --git a/ielts_be/controllers/impl/exam/writing.py b/ielts_be/controllers/impl/exam/writing.py new file mode 100644 index 0000000..75bcc17 --- /dev/null +++ b/ielts_be/controllers/impl/exam/writing.py @@ -0,0 +1,19 @@ +from fastapi import UploadFile, HTTPException + +from ielts_be.controllers import IWritingController +from ielts_be.services import IWritingService + + +class WritingController(IWritingController): + + def __init__(self, writing_service: IWritingService): + self._service = writing_service + + async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): + return await self._service.get_writing_task_general_question(task, topic, difficulty) + + async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: str): + if attachment.content_type not in ['image/jpeg', 'image/png']: + raise HTTPException(status_code=400, detail="Invalid file type. Only JPEG and PNG allowed.") + + return await self._service.get_writing_task_academic_question(task, attachment, difficulty) diff --git a/app/controllers/impl/grade.py b/ielts_be/controllers/impl/grade.py similarity index 86% rename from app/controllers/impl/grade.py rename to ielts_be/controllers/impl/grade.py index cae2087..830cbef 100644 --- a/app/controllers/impl/grade.py +++ b/ielts_be/controllers/impl/grade.py @@ -1,15 +1,14 @@ import logging -from typing import Dict, List, Union -from uuid import uuid4 +from typing import Dict from fastapi import BackgroundTasks, Response, HTTPException from fastapi.datastructures import FormData -from app.controllers.abc import IGradeController -from app.dtos.evaluation import EvaluationType -from app.dtos.speaking import GradeSpeakingItem -from app.dtos.writing import WritingGradeTaskDTO -from app.services.abc import IGradeService, IEvaluationService +from ielts_be.controllers import IGradeController +from ielts_be.services import IGradeService, IEvaluationService +from ielts_be.dtos.evaluation import EvaluationType +from ielts_be.dtos.speaking import GradeSpeakingItem +from ielts_be.dtos.writing import WritingGradeTaskDTO class GradeController(IGradeController): @@ -26,6 +25,9 @@ class GradeController(IGradeController): self, task: int, dto: WritingGradeTaskDTO, background_tasks: BackgroundTasks ): + if task == 1 and dto.type == "academic" and dto.attachment is None: + raise HTTPException(status_code=400, detail="Academic writing task requires a picture!") + await self._evaluation_service.create_evaluation( dto.userId, dto.sessionId, dto.exerciseId, EvaluationType.WRITING, task ) diff --git a/app/controllers/impl/training.py b/ielts_be/controllers/impl/training.py similarity index 73% rename from app/controllers/impl/training.py rename to ielts_be/controllers/impl/training.py index dab335d..951e24a 100644 --- a/app/controllers/impl/training.py +++ b/ielts_be/controllers/impl/training.py @@ -1,8 +1,8 @@ from typing import Dict -from app.controllers.abc import ITrainingController -from app.dtos.training import FetchTipsDTO -from app.services.abc import ITrainingService +from ielts_be.controllers import ITrainingController +from ielts_be.services import ITrainingService +from ielts_be.dtos.training import FetchTipsDTO class TrainingController(ITrainingController): diff --git a/app/controllers/impl/user.py b/ielts_be/controllers/impl/user.py similarity index 60% rename from app/controllers/impl/user.py rename to ielts_be/controllers/impl/user.py index ffdbbec..9802e5d 100644 --- a/app/controllers/impl/user.py +++ b/ielts_be/controllers/impl/user.py @@ -1,6 +1,6 @@ -from app.controllers.abc import IUserController -from app.dtos.user_batch import BatchUsersDTO -from app.services.abc import IUserService +from ielts_be.controllers import IUserController +from ielts_be.services import IUserService +from ielts_be.dtos.user_batch import BatchUsersDTO class UserController(IUserController): diff --git a/app/__init__.py b/ielts_be/dtos/__init__.py similarity index 100% rename from app/__init__.py rename to ielts_be/dtos/__init__.py diff --git a/app/dtos/evaluation.py b/ielts_be/dtos/evaluation.py similarity index 100% rename from app/dtos/evaluation.py rename to ielts_be/dtos/evaluation.py diff --git a/app/controllers/__init__.py b/ielts_be/dtos/exams/__init__.py similarity index 100% rename from app/controllers/__init__.py rename to ielts_be/dtos/exams/__init__.py diff --git a/app/dtos/exams/level.py b/ielts_be/dtos/exams/level.py similarity index 100% rename from app/dtos/exams/level.py rename to ielts_be/dtos/exams/level.py diff --git a/app/dtos/exams/listening.py b/ielts_be/dtos/exams/listening.py similarity index 100% rename from app/dtos/exams/listening.py rename to ielts_be/dtos/exams/listening.py diff --git a/app/dtos/exams/reading.py b/ielts_be/dtos/exams/reading.py similarity index 100% rename from app/dtos/exams/reading.py rename to ielts_be/dtos/exams/reading.py diff --git a/app/dtos/level.py b/ielts_be/dtos/level.py similarity index 87% rename from app/dtos/level.py rename to ielts_be/dtos/level.py index 1987a75..e95f843 100644 --- a/app/dtos/level.py +++ b/ielts_be/dtos/level.py @@ -2,7 +2,7 @@ from typing import List, Optional from pydantic import BaseModel -from app.configs.constants import LevelExerciseType +from ielts_be.configs.constants import LevelExerciseType class LevelExercises(BaseModel): diff --git a/app/dtos/listening.py b/ielts_be/dtos/listening.py similarity index 86% rename from app/dtos/listening.py rename to ielts_be/dtos/listening.py index 8c98434..8577d3f 100644 --- a/app/dtos/listening.py +++ b/ielts_be/dtos/listening.py @@ -4,7 +4,7 @@ from typing import List, Dict, Optional from pydantic import BaseModel, Field -from app.configs.constants import MinTimers, EducationalContent, ListeningExerciseType +from ielts_be.configs.constants import MinTimers, EducationalContent, ListeningExerciseType class SaveListeningDTO(BaseModel): diff --git a/app/dtos/reading.py b/ielts_be/dtos/reading.py similarity index 84% rename from app/dtos/reading.py rename to ielts_be/dtos/reading.py index b5907e8..2de9085 100644 --- a/app/dtos/reading.py +++ b/ielts_be/dtos/reading.py @@ -3,7 +3,7 @@ from typing import List, Optional from pydantic import BaseModel, Field -from app.configs.constants import ReadingExerciseType, EducationalContent +from ielts_be.configs.constants import ReadingExerciseType, EducationalContent class ReadingExercise(BaseModel): type: ReadingExerciseType diff --git a/app/dtos/sheet.py b/ielts_be/dtos/sheet.py similarity index 100% rename from app/dtos/sheet.py rename to ielts_be/dtos/sheet.py diff --git a/app/dtos/speaking.py b/ielts_be/dtos/speaking.py similarity index 81% rename from app/dtos/speaking.py rename to ielts_be/dtos/speaking.py index 859f77e..d9c0906 100644 --- a/app/dtos/speaking.py +++ b/ielts_be/dtos/speaking.py @@ -1,5 +1,3 @@ -from typing import List, Dict - from fastapi import UploadFile from pydantic import BaseModel diff --git a/app/dtos/training.py b/ielts_be/dtos/training.py similarity index 100% rename from app/dtos/training.py rename to ielts_be/dtos/training.py diff --git a/app/dtos/user_batch.py b/ielts_be/dtos/user_batch.py similarity index 100% rename from app/dtos/user_batch.py rename to ielts_be/dtos/user_batch.py diff --git a/app/dtos/video.py b/ielts_be/dtos/video.py similarity index 95% rename from app/dtos/video.py rename to ielts_be/dtos/video.py index a827e6b..af3c9af 100644 --- a/app/dtos/video.py +++ b/ielts_be/dtos/video.py @@ -1,4 +1,3 @@ -import random from enum import Enum from typing import Optional diff --git a/app/dtos/writing.py b/ielts_be/dtos/writing.py similarity index 65% rename from app/dtos/writing.py rename to ielts_be/dtos/writing.py index 94c662d..ba33aa1 100644 --- a/app/dtos/writing.py +++ b/ielts_be/dtos/writing.py @@ -1,3 +1,5 @@ +from typing import Optional + from pydantic import BaseModel @@ -7,3 +9,5 @@ class WritingGradeTaskDTO(BaseModel): exerciseId: str question: str answer: str + type: str + attachment: Optional[str] diff --git a/app/exceptions/__init__.py b/ielts_be/exceptions/__init__.py similarity index 100% rename from app/exceptions/__init__.py rename to ielts_be/exceptions/__init__.py diff --git a/app/exceptions/exceptions.py b/ielts_be/exceptions/exceptions.py similarity index 100% rename from app/exceptions/exceptions.py rename to ielts_be/exceptions/exceptions.py diff --git a/app/helpers/__init__.py b/ielts_be/helpers/__init__.py similarity index 100% rename from app/helpers/__init__.py rename to ielts_be/helpers/__init__.py diff --git a/app/helpers/exercises.py b/ielts_be/helpers/exercises.py similarity index 100% rename from app/helpers/exercises.py rename to ielts_be/helpers/exercises.py diff --git a/app/helpers/file.py b/ielts_be/helpers/file.py similarity index 92% rename from app/helpers/file.py rename to ielts_be/helpers/file.py index 079e42e..40403b8 100644 --- a/app/helpers/file.py +++ b/ielts_be/helpers/file.py @@ -117,3 +117,9 @@ class FileHelper: await file.write(file_bytes) return ext, path_id + + @staticmethod + async def encode_image(image_path: str) -> str: + async with aiofiles.open(image_path, "rb") as image_file: + img = await image_file.read() + return base64.b64encode(img).decode('utf-8') diff --git a/app/helpers/text.py b/ielts_be/helpers/text.py similarity index 100% rename from app/helpers/text.py rename to ielts_be/helpers/text.py diff --git a/app/helpers/token_counter.py b/ielts_be/helpers/token_counter.py similarity index 100% rename from app/helpers/token_counter.py rename to ielts_be/helpers/token_counter.py diff --git a/app/mappers/__init__.py b/ielts_be/mappers/__init__.py similarity index 100% rename from app/mappers/__init__.py rename to ielts_be/mappers/__init__.py diff --git a/app/mappers/level.py b/ielts_be/mappers/level.py similarity index 92% rename from app/mappers/level.py rename to ielts_be/mappers/level.py index 2863f73..b54abcf 100644 --- a/app/mappers/level.py +++ b/ielts_be/mappers/level.py @@ -2,12 +2,12 @@ from typing import Dict, Any from pydantic import ValidationError -from app.dtos.exams.level import ( +from ielts_be.dtos.exams.level import ( MultipleChoiceExercise, FillBlanksExercise, Part, Exam, Text ) -from app.dtos.sheet import Sheet, Option, MultipleChoiceQuestion, FillBlanksWord +from ielts_be.dtos.sheet import Sheet, Option, MultipleChoiceQuestion, FillBlanksWord class LevelMapper: diff --git a/app/mappers/listening.py b/ielts_be/mappers/listening.py similarity index 98% rename from app/mappers/listening.py rename to ielts_be/mappers/listening.py index ab0b3b5..e243549 100644 --- a/app/mappers/listening.py +++ b/ielts_be/mappers/listening.py @@ -2,7 +2,7 @@ from typing import Dict, Any, List, Union, Optional from pydantic import BaseModel -from app.dtos.exams.listening import ( +from ielts_be.dtos.exams.listening import ( TrueFalseExercise, MultipleChoiceExercise, WriteBlanksExercise, diff --git a/app/mappers/reading.py b/ielts_be/mappers/reading.py similarity index 97% rename from app/mappers/reading.py rename to ielts_be/mappers/reading.py index 8065dab..bf67e8e 100644 --- a/app/mappers/reading.py +++ b/ielts_be/mappers/reading.py @@ -1,6 +1,6 @@ from typing import Dict, Any -from app.dtos.exams.reading import ( +from ielts_be.dtos.exams.reading import ( Part, Exam, Context, FillBlanksExercise, TrueFalseExercise, MatchSentencesExercise, WriteBlanksExercise, MultipleChoice diff --git a/app/middlewares/__init__.py b/ielts_be/middlewares/__init__.py similarity index 100% rename from app/middlewares/__init__.py rename to ielts_be/middlewares/__init__.py diff --git a/app/middlewares/authentication.py b/ielts_be/middlewares/authentication.py similarity index 100% rename from app/middlewares/authentication.py rename to ielts_be/middlewares/authentication.py diff --git a/app/middlewares/authorization.py b/ielts_be/middlewares/authorization.py similarity index 90% rename from app/middlewares/authorization.py rename to ielts_be/middlewares/authorization.py index e0c95d1..1a6d949 100644 --- a/app/middlewares/authorization.py +++ b/ielts_be/middlewares/authorization.py @@ -5,7 +5,7 @@ from fastapi import Request from fastapi.openapi.models import APIKey, APIKeyIn from fastapi.security.base import SecurityBase -from app.exceptions import CustomException, UnauthorizedException +from ielts_be.exceptions import CustomException, UnauthorizedException class BaseAuthorization(ABC): diff --git a/ielts_be/repositories/__init__.py b/ielts_be/repositories/__init__.py new file mode 100644 index 0000000..621885c --- /dev/null +++ b/ielts_be/repositories/__init__.py @@ -0,0 +1,3 @@ +from .abc import * + +__all__ = abc.__all__ diff --git a/app/repositories/abc/__init__.py b/ielts_be/repositories/abc/__init__.py similarity index 100% rename from app/repositories/abc/__init__.py rename to ielts_be/repositories/abc/__init__.py diff --git a/app/repositories/abc/document_store.py b/ielts_be/repositories/abc/document_store.py similarity index 100% rename from app/repositories/abc/document_store.py rename to ielts_be/repositories/abc/document_store.py diff --git a/app/repositories/abc/file_storage.py b/ielts_be/repositories/abc/file_storage.py similarity index 100% rename from app/repositories/abc/file_storage.py rename to ielts_be/repositories/abc/file_storage.py diff --git a/ielts_be/repositories/impl/__init__.py b/ielts_be/repositories/impl/__init__.py new file mode 100644 index 0000000..fee1e34 --- /dev/null +++ b/ielts_be/repositories/impl/__init__.py @@ -0,0 +1,6 @@ +from .document_stores import * +from .file_storage import * + +__all__ = [] +__all__.extend(document_stores.__all__) +__all__.extend(file_storage.__all__) diff --git a/app/repositories/impl/document_stores/__init__.py b/ielts_be/repositories/impl/document_stores/__init__.py similarity index 56% rename from app/repositories/impl/document_stores/__init__.py rename to ielts_be/repositories/impl/document_stores/__init__.py index ccea6ee..abc32e3 100644 --- a/app/repositories/impl/document_stores/__init__.py +++ b/ielts_be/repositories/impl/document_stores/__init__.py @@ -1,7 +1,7 @@ from .firestore import Firestore -#from .mongo import MongoDB +from .mongo import MongoDB __all__ = [ "Firestore", - #"MongoDB" + "MongoDB" ] diff --git a/app/repositories/impl/document_stores/firestore.py b/ielts_be/repositories/impl/document_stores/firestore.py similarity index 95% rename from app/repositories/impl/document_stores/firestore.py rename to ielts_be/repositories/impl/document_stores/firestore.py index 2facebb..e3d0a8a 100644 --- a/app/repositories/impl/document_stores/firestore.py +++ b/ielts_be/repositories/impl/document_stores/firestore.py @@ -4,7 +4,7 @@ from typing import Optional, List, Dict from google.cloud.firestore_v1.async_client import AsyncClient from google.cloud.firestore_v1.async_collection import AsyncCollectionReference from google.cloud.firestore_v1.async_document import AsyncDocumentReference -from app.repositories.abc import IDocumentStore +from ielts_be.repositories import IDocumentStore class Firestore(IDocumentStore): diff --git a/app/repositories/impl/document_stores/mongo.py b/ielts_be/repositories/impl/document_stores/mongo.py similarity index 93% rename from app/repositories/impl/document_stores/mongo.py rename to ielts_be/repositories/impl/document_stores/mongo.py index cef2552..ecbec65 100644 --- a/app/repositories/impl/document_stores/mongo.py +++ b/ielts_be/repositories/impl/document_stores/mongo.py @@ -4,7 +4,7 @@ from typing import Optional, List, Dict from motor.motor_asyncio import AsyncIOMotorDatabase -from app.repositories.abc import IDocumentStore +from ielts_be.repositories import IDocumentStore class MongoDB(IDocumentStore): diff --git a/app/repositories/impl/file_storage/__init__.py b/ielts_be/repositories/impl/file_storage/__init__.py similarity index 100% rename from app/repositories/impl/file_storage/__init__.py rename to ielts_be/repositories/impl/file_storage/__init__.py diff --git a/app/repositories/impl/file_storage/firebase.py b/ielts_be/repositories/impl/file_storage/firebase.py similarity index 96% rename from app/repositories/impl/file_storage/firebase.py rename to ielts_be/repositories/impl/file_storage/firebase.py index 9450afe..8999c7a 100644 --- a/app/repositories/impl/file_storage/firebase.py +++ b/ielts_be/repositories/impl/file_storage/firebase.py @@ -4,7 +4,7 @@ from typing import Optional import aiofiles from httpx import AsyncClient -from app.repositories.abc import IFileStorage +from ielts_be.repositories import IFileStorage class FirebaseStorage(IFileStorage): diff --git a/ielts_be/services/__init__.py b/ielts_be/services/__init__.py new file mode 100644 index 0000000..621885c --- /dev/null +++ b/ielts_be/services/__init__.py @@ -0,0 +1,3 @@ +from .abc import * + +__all__ = abc.__all__ diff --git a/app/services/abc/__init__.py b/ielts_be/services/abc/__init__.py similarity index 100% rename from app/services/abc/__init__.py rename to ielts_be/services/abc/__init__.py diff --git a/app/services/abc/evaluation.py b/ielts_be/services/abc/evaluation.py similarity index 87% rename from app/services/abc/evaluation.py rename to ielts_be/services/abc/evaluation.py index 0347859..9dbb48a 100644 --- a/app/services/abc/evaluation.py +++ b/ielts_be/services/abc/evaluation.py @@ -1,9 +1,8 @@ from abc import abstractmethod, ABC -from typing import Union, List, Dict from fastapi import BackgroundTasks -from app.dtos.evaluation import EvaluationType +from ielts_be.dtos.evaluation import EvaluationType class IEvaluationService(ABC): diff --git a/app/services/abc/exam/__init__.py b/ielts_be/services/abc/exam/__init__.py similarity index 100% rename from app/services/abc/exam/__init__.py rename to ielts_be/services/abc/exam/__init__.py diff --git a/app/services/abc/exam/exercises.py b/ielts_be/services/abc/exam/exercises.py similarity index 100% rename from app/services/abc/exam/exercises.py rename to ielts_be/services/abc/exam/exercises.py diff --git a/app/services/abc/exam/grade.py b/ielts_be/services/abc/exam/grade.py similarity index 100% rename from app/services/abc/exam/grade.py rename to ielts_be/services/abc/exam/grade.py diff --git a/app/services/abc/exam/level.py b/ielts_be/services/abc/exam/level.py similarity index 90% rename from app/services/abc/exam/level.py rename to ielts_be/services/abc/exam/level.py index 9ef0886..c056b43 100644 --- a/app/services/abc/exam/level.py +++ b/ielts_be/services/abc/exam/level.py @@ -1,12 +1,7 @@ from abc import ABC, abstractmethod -import random - from typing import Dict, Optional - from fastapi import UploadFile -from app.configs.constants import EducationalContent - class ILevelService(ABC): diff --git a/app/services/abc/exam/listening.py b/ielts_be/services/abc/exam/listening.py similarity index 100% rename from app/services/abc/exam/listening.py rename to ielts_be/services/abc/exam/listening.py diff --git a/app/services/abc/exam/reading.py b/ielts_be/services/abc/exam/reading.py similarity index 100% rename from app/services/abc/exam/reading.py rename to ielts_be/services/abc/exam/reading.py diff --git a/app/services/abc/exam/speaking.py b/ielts_be/services/abc/exam/speaking.py similarity index 100% rename from app/services/abc/exam/speaking.py rename to ielts_be/services/abc/exam/speaking.py diff --git a/app/services/abc/exam/writing.py b/ielts_be/services/abc/exam/writing.py similarity index 52% rename from app/services/abc/exam/writing.py rename to ielts_be/services/abc/exam/writing.py index 3f9c47e..94d223d 100644 --- a/app/services/abc/exam/writing.py +++ b/ielts_be/services/abc/exam/writing.py @@ -1,4 +1,8 @@ from abc import ABC, abstractmethod +from typing import Optional + +from fastapi import UploadFile + class IWritingService(ABC): @@ -7,5 +11,9 @@ class IWritingService(ABC): pass @abstractmethod - async def grade_writing_task(self, task: int, question: str, answer: str): + async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: str): + pass + + @abstractmethod + async def grade_writing_task(self, task: int, question: str, answer: str, attachment: Optional[str]): pass diff --git a/app/services/abc/third_parties/__init__.py b/ielts_be/services/abc/third_parties/__init__.py similarity index 100% rename from app/services/abc/third_parties/__init__.py rename to ielts_be/services/abc/third_parties/__init__.py diff --git a/app/services/abc/third_parties/ai_detector.py b/ielts_be/services/abc/third_parties/ai_detector.py similarity index 100% rename from app/services/abc/third_parties/ai_detector.py rename to ielts_be/services/abc/third_parties/ai_detector.py diff --git a/app/services/abc/third_parties/llm.py b/ielts_be/services/abc/third_parties/llm.py similarity index 100% rename from app/services/abc/third_parties/llm.py rename to ielts_be/services/abc/third_parties/llm.py diff --git a/app/services/abc/third_parties/stt.py b/ielts_be/services/abc/third_parties/stt.py similarity index 100% rename from app/services/abc/third_parties/stt.py rename to ielts_be/services/abc/third_parties/stt.py diff --git a/app/services/abc/third_parties/tts.py b/ielts_be/services/abc/third_parties/tts.py similarity index 100% rename from app/services/abc/third_parties/tts.py rename to ielts_be/services/abc/third_parties/tts.py diff --git a/app/services/abc/third_parties/vid_gen.py b/ielts_be/services/abc/third_parties/vid_gen.py similarity index 78% rename from app/services/abc/third_parties/vid_gen.py rename to ielts_be/services/abc/third_parties/vid_gen.py index 6851abf..e31f89b 100644 --- a/app/services/abc/third_parties/vid_gen.py +++ b/ielts_be/services/abc/third_parties/vid_gen.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Dict, List +from typing import Dict, List, Optional class IVideoGeneratorService(ABC): @@ -14,7 +14,7 @@ class IVideoGeneratorService(ABC): ] @abstractmethod - async def create_video(self, text: str, avatar: str, title: str): + async def create_video(self, text: str, avatar: str): pass @abstractmethod diff --git a/app/services/abc/training/__init__.py b/ielts_be/services/abc/training/__init__.py similarity index 100% rename from app/services/abc/training/__init__.py rename to ielts_be/services/abc/training/__init__.py diff --git a/app/services/abc/training/kb.py b/ielts_be/services/abc/training/kb.py similarity index 100% rename from app/services/abc/training/kb.py rename to ielts_be/services/abc/training/kb.py diff --git a/app/services/abc/training/training.py b/ielts_be/services/abc/training/training.py similarity index 100% rename from app/services/abc/training/training.py rename to ielts_be/services/abc/training/training.py diff --git a/app/services/abc/user.py b/ielts_be/services/abc/user.py similarity index 71% rename from app/services/abc/user.py rename to ielts_be/services/abc/user.py index 10f4886..bd01a75 100644 --- a/app/services/abc/user.py +++ b/ielts_be/services/abc/user.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from app.dtos.user_batch import BatchUsersDTO +from ielts_be.dtos.user_batch import BatchUsersDTO class IUserService(ABC): diff --git a/app/services/impl/__init__.py b/ielts_be/services/impl/__init__.py similarity index 100% rename from app/services/impl/__init__.py rename to ielts_be/services/impl/__init__.py diff --git a/app/services/impl/exam/__init__.py b/ielts_be/services/impl/exam/__init__.py similarity index 81% rename from app/services/impl/exam/__init__.py rename to ielts_be/services/impl/exam/__init__.py index fd3ebc8..5a0d654 100644 --- a/app/services/impl/exam/__init__.py +++ b/ielts_be/services/impl/exam/__init__.py @@ -4,6 +4,7 @@ from .reading import ReadingService from .speaking import SpeakingService from .writing import WritingService from .grade import GradeService +from .evaluation import EvaluationService __all__ = [ @@ -13,4 +14,5 @@ __all__ = [ "SpeakingService", "WritingService", "GradeService", + "EvaluationService" ] diff --git a/app/services/impl/exam/evaluation.py b/ielts_be/services/impl/exam/evaluation.py similarity index 88% rename from app/services/impl/exam/evaluation.py rename to ielts_be/services/impl/exam/evaluation.py index 2d0a94c..013f252 100644 --- a/app/services/impl/exam/evaluation.py +++ b/ielts_be/services/impl/exam/evaluation.py @@ -1,13 +1,13 @@ import logging -from typing import Union, Dict, List +from typing import Union, List from fastapi import BackgroundTasks -from app.dtos.evaluation import EvaluationType -from app.dtos.speaking import GradeSpeakingItem -from app.dtos.writing import WritingGradeTaskDTO -from app.repositories.abc import IDocumentStore -from app.services.abc import IWritingService, ISpeakingService, IEvaluationService +from ielts_be.dtos.evaluation import EvaluationType +from ielts_be.dtos.speaking import GradeSpeakingItem +from ielts_be.dtos.writing import WritingGradeTaskDTO +from ielts_be.repositories import IDocumentStore +from ielts_be.services import IWritingService, ISpeakingService, IEvaluationService class EvaluationService(IEvaluationService): @@ -62,7 +62,8 @@ class EvaluationService(IEvaluationService): result = await self._writing_service.grade_writing_task( task, solution.question, - solution.answer + solution.answer, + solution.attachment ) else: result = await self._speaking_service.grade_speaking_task( diff --git a/app/services/impl/exam/grade.py b/ielts_be/services/impl/exam/grade.py similarity index 96% rename from app/services/impl/exam/grade.py rename to ielts_be/services/impl/exam/grade.py index eaabd1f..6685f33 100644 --- a/app/services/impl/exam/grade.py +++ b/ielts_be/services/impl/exam/grade.py @@ -1,8 +1,8 @@ import json from typing import List, Dict -from app.configs.constants import GPTModels, TemperatureSettings -from app.services.abc import ILLMService, IGradeService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.services import ILLMService, IGradeService class GradeService(IGradeService): diff --git a/app/services/impl/exam/level/__init__.py b/ielts_be/services/impl/exam/level/__init__.py similarity index 95% rename from app/services/impl/exam/level/__init__.py rename to ielts_be/services/impl/exam/level/__init__.py index c93f71b..6ff23cb 100644 --- a/app/services/impl/exam/level/__init__.py +++ b/ielts_be/services/impl/exam/level/__init__.py @@ -6,10 +6,10 @@ from fastapi import UploadFile import random -from app.configs.constants import EducationalContent -from app.dtos.level import LevelExercisesDTO -from app.repositories.abc import IDocumentStore -from app.services.abc import ( +from ielts_be.configs.constants import EducationalContent +from ielts_be.dtos.level import LevelExercisesDTO +from ielts_be.repositories import IDocumentStore +from ielts_be.services import ( ILevelService, ILLMService, IReadingService, IWritingService, IListeningService, ISpeakingService ) diff --git a/app/services/impl/exam/level/exercises/__init__.py b/ielts_be/services/impl/exam/level/exercises/__init__.py similarity index 85% rename from app/services/impl/exam/level/exercises/__init__.py rename to ielts_be/services/impl/exam/level/exercises/__init__.py index 1d09802..a7778ec 100644 --- a/app/services/impl/exam/level/exercises/__init__.py +++ b/ielts_be/services/impl/exam/level/exercises/__init__.py @@ -1,7 +1,7 @@ from .multiple_choice import MultipleChoice from .blank_space import BlankSpace from .passage_utas import PassageUtas -from .fillBlanks import FillBlanks +from .fill_blanks import FillBlanks __all__ = [ "MultipleChoice", diff --git a/app/services/impl/exam/level/exercises/blank_space.py b/ielts_be/services/impl/exam/level/exercises/blank_space.py similarity index 92% rename from app/services/impl/exam/level/exercises/blank_space.py rename to ielts_be/services/impl/exam/level/exercises/blank_space.py index 5758f1f..fe39fc7 100644 --- a/app/services/impl/exam/level/exercises/blank_space.py +++ b/ielts_be/services/impl/exam/level/exercises/blank_space.py @@ -1,7 +1,7 @@ import random -from app.configs.constants import EducationalContent, GPTModels, TemperatureSettings -from app.services.abc import ILLMService +from ielts_be.configs.constants import EducationalContent, GPTModels, TemperatureSettings +from ielts_be.services import ILLMService class BlankSpace: diff --git a/app/services/impl/exam/level/exercises/fillBlanks.py b/ielts_be/services/impl/exam/level/exercises/fill_blanks.py similarity index 94% rename from app/services/impl/exam/level/exercises/fillBlanks.py rename to ielts_be/services/impl/exam/level/exercises/fill_blanks.py index 32ad6c4..976aa56 100644 --- a/app/services/impl/exam/level/exercises/fillBlanks.py +++ b/ielts_be/services/impl/exam/level/exercises/fill_blanks.py @@ -1,7 +1,7 @@ import random -from app.configs.constants import GPTModels, TemperatureSettings, EducationalContent -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings, EducationalContent +from ielts_be.services import ILLMService class FillBlanks: diff --git a/app/services/impl/exam/level/exercises/multiple_choice.py b/ielts_be/services/impl/exam/level/exercises/multiple_choice.py similarity index 95% rename from app/services/impl/exam/level/exercises/multiple_choice.py rename to ielts_be/services/impl/exam/level/exercises/multiple_choice.py index fe45819..ea4e129 100644 --- a/app/services/impl/exam/level/exercises/multiple_choice.py +++ b/ielts_be/services/impl/exam/level/exercises/multiple_choice.py @@ -1,6 +1,6 @@ -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class MultipleChoice: diff --git a/app/services/impl/exam/level/exercises/passage_utas.py b/ielts_be/services/impl/exam/level/exercises/passage_utas.py similarity index 94% rename from app/services/impl/exam/level/exercises/passage_utas.py rename to ielts_be/services/impl/exam/level/exercises/passage_utas.py index ba16a66..683cee4 100644 --- a/app/services/impl/exam/level/exercises/passage_utas.py +++ b/ielts_be/services/impl/exam/level/exercises/passage_utas.py @@ -1,8 +1,8 @@ from typing import Optional -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService, IReadingService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService, IReadingService class PassageUtas: diff --git a/app/services/impl/exam/level/full_exams/__init__.py b/ielts_be/services/impl/exam/level/full_exams/__init__.py similarity index 100% rename from app/services/impl/exam/level/full_exams/__init__.py rename to ielts_be/services/impl/exam/level/full_exams/__init__.py diff --git a/app/services/impl/exam/level/full_exams/custom.py b/ielts_be/services/impl/exam/level/full_exams/custom.py similarity index 99% rename from app/services/impl/exam/level/full_exams/custom.py rename to ielts_be/services/impl/exam/level/full_exams/custom.py index b9f6668..1b1492d 100644 --- a/app/services/impl/exam/level/full_exams/custom.py +++ b/ielts_be/services/impl/exam/level/full_exams/custom.py @@ -3,8 +3,8 @@ import random from typing import Dict -from app.configs.constants import CustomLevelExerciseTypes, EducationalContent -from app.services.abc import ( +from ielts_be.configs.constants import CustomLevelExerciseTypes, EducationalContent +from ielts_be.services import ( ILLMService, ILevelService, IReadingService, IWritingService, IListeningService, ISpeakingService ) diff --git a/app/services/impl/exam/level/full_exams/level_utas.py b/ielts_be/services/impl/exam/level/full_exams/level_utas.py similarity index 98% rename from app/services/impl/exam/level/full_exams/level_utas.py rename to ielts_be/services/impl/exam/level/full_exams/level_utas.py index dc3c161..6234524 100644 --- a/app/services/impl/exam/level/full_exams/level_utas.py +++ b/ielts_be/services/impl/exam/level/full_exams/level_utas.py @@ -1,7 +1,7 @@ import json import uuid -from app.services.abc import ILLMService +from ielts_be.services import ILLMService class LevelUtas: diff --git a/app/services/impl/exam/level/mc_variants.json b/ielts_be/services/impl/exam/level/mc_variants.json similarity index 100% rename from app/services/impl/exam/level/mc_variants.json rename to ielts_be/services/impl/exam/level/mc_variants.json diff --git a/app/services/impl/exam/level/upload.py b/ielts_be/services/impl/exam/level/upload.py similarity index 96% rename from app/services/impl/exam/level/upload.py rename to ielts_be/services/impl/exam/level/upload.py index 365d0bf..7b4004b 100644 --- a/app/services/impl/exam/level/upload.py +++ b/ielts_be/services/impl/exam/level/upload.py @@ -9,13 +9,13 @@ from typing import Dict, Any, Optional import pdfplumber from fastapi import UploadFile -from app.services.abc import ILLMService -from app.helpers import FileHelper -from app.mappers import LevelMapper +from ielts_be.services import ILLMService +from ielts_be.helpers import FileHelper +from ielts_be.mappers import LevelMapper -from app.dtos.exams.level import Exam -from app.dtos.sheet import Sheet -from app.utils import suppress_loggers +from ielts_be.dtos.exams.level import Exam +from ielts_be.dtos.sheet import Sheet +from ielts_be.utils import suppress_loggers class UploadLevelModule: diff --git a/app/services/impl/exam/listening/__init__.py b/ielts_be/services/impl/exam/listening/__init__.py similarity index 97% rename from app/services/impl/exam/listening/__init__.py rename to ielts_be/services/impl/exam/listening/__init__.py index fe319be..355b2f0 100644 --- a/app/services/impl/exam/listening/__init__.py +++ b/ielts_be/services/impl/exam/listening/__init__.py @@ -5,14 +5,14 @@ from typing import Dict, Any from starlette.datastructures import UploadFile -from app.dtos.listening import GenerateListeningExercises, Dialog, ListeningExercises -from app.repositories.abc import IFileStorage, IDocumentStore -from app.services.abc import IListeningService, ILLMService, ITextToSpeechService, ISpeechToTextService -from app.configs.constants import ( +from ielts_be.dtos.listening import GenerateListeningExercises, Dialog, ListeningExercises +from ielts_be.repositories import IFileStorage, IDocumentStore +from ielts_be.services import IListeningService, ILLMService, ITextToSpeechService, ISpeechToTextService +from ielts_be.configs.constants import ( NeuralVoices, GPTModels, TemperatureSettings, EducationalContent, FieldsAndExercises ) -from app.helpers import FileHelper +from ielts_be.helpers import FileHelper from .import_listening import ImportListeningModule from .write_blank_forms import WriteBlankForms from .write_blanks import WriteBlanks diff --git a/app/services/impl/exam/listening/import_listening.py b/ielts_be/services/impl/exam/listening/import_listening.py similarity index 97% rename from app/services/impl/exam/listening/import_listening.py rename to ielts_be/services/impl/exam/listening/import_listening.py index af330d4..377c17c 100644 --- a/app/services/impl/exam/listening/import_listening.py +++ b/ielts_be/services/impl/exam/listening/import_listening.py @@ -5,10 +5,10 @@ from uuid import uuid4 import aiofiles from fastapi import UploadFile -from app.dtos.exams.listening import ListeningExam -from app.helpers import FileHelper -from app.mappers.listening import ListeningMapper -from app.services.abc import ILLMService +from ielts_be.dtos.exams.listening import ListeningExam +from ielts_be.helpers import FileHelper +from ielts_be.mappers.listening import ListeningMapper +from ielts_be.services import ILLMService class ImportListeningModule: diff --git a/app/services/impl/exam/listening/write_blank_forms.py b/ielts_be/services/impl/exam/listening/write_blank_forms.py similarity index 91% rename from app/services/impl/exam/listening/write_blank_forms.py rename to ielts_be/services/impl/exam/listening/write_blank_forms.py index 11b7339..5ad23b7 100644 --- a/app/services/impl/exam/listening/write_blank_forms.py +++ b/ielts_be/services/impl/exam/listening/write_blank_forms.py @@ -1,8 +1,8 @@ import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class WriteBlankForms: diff --git a/app/services/impl/exam/listening/write_blank_notes.py b/ielts_be/services/impl/exam/listening/write_blank_notes.py similarity index 93% rename from app/services/impl/exam/listening/write_blank_notes.py rename to ielts_be/services/impl/exam/listening/write_blank_notes.py index c54448f..83e55f2 100644 --- a/app/services/impl/exam/listening/write_blank_notes.py +++ b/ielts_be/services/impl/exam/listening/write_blank_notes.py @@ -1,8 +1,8 @@ import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class WriteBlankNotes: diff --git a/app/services/impl/exam/listening/write_blanks.py b/ielts_be/services/impl/exam/listening/write_blanks.py similarity index 90% rename from app/services/impl/exam/listening/write_blanks.py rename to ielts_be/services/impl/exam/listening/write_blanks.py index 0049068..c848a93 100644 --- a/app/services/impl/exam/listening/write_blanks.py +++ b/ielts_be/services/impl/exam/listening/write_blanks.py @@ -1,8 +1,8 @@ import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class WriteBlanks: diff --git a/app/services/impl/exam/reading/__init__.py b/ielts_be/services/impl/exam/reading/__init__.py similarity index 96% rename from app/services/impl/exam/reading/__init__.py rename to ielts_be/services/impl/exam/reading/__init__.py index 57a4312..aa39b0b 100644 --- a/app/services/impl/exam/reading/__init__.py +++ b/ielts_be/services/impl/exam/reading/__init__.py @@ -3,10 +3,10 @@ from logging import getLogger from fastapi import UploadFile -from app.configs.constants import GPTModels, FieldsAndExercises, TemperatureSettings -from app.dtos.reading import ReadingDTO -from app.helpers import ExercisesHelper -from app.services.abc import IReadingService, ILLMService +from ielts_be.configs.constants import GPTModels, FieldsAndExercises, TemperatureSettings +from ielts_be.dtos.reading import ReadingDTO +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import IReadingService, ILLMService from .fill_blanks import FillBlanks from .idea_match import IdeaMatch from .paragraph_match import ParagraphMatch diff --git a/app/services/impl/exam/reading/fill_blanks.py b/ielts_be/services/impl/exam/reading/fill_blanks.py similarity index 93% rename from app/services/impl/exam/reading/fill_blanks.py rename to ielts_be/services/impl/exam/reading/fill_blanks.py index 5888a77..b6da12a 100644 --- a/app/services/impl/exam/reading/fill_blanks.py +++ b/ielts_be/services/impl/exam/reading/fill_blanks.py @@ -1,8 +1,8 @@ import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class FillBlanks: diff --git a/app/services/impl/exam/reading/idea_match.py b/ielts_be/services/impl/exam/reading/idea_match.py similarity index 90% rename from app/services/impl/exam/reading/idea_match.py rename to ielts_be/services/impl/exam/reading/idea_match.py index e2bcaa0..44486f0 100644 --- a/app/services/impl/exam/reading/idea_match.py +++ b/ielts_be/services/impl/exam/reading/idea_match.py @@ -1,8 +1,8 @@ import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class IdeaMatch: diff --git a/app/services/impl/exam/reading/import_reading.py b/ielts_be/services/impl/exam/reading/import_reading.py similarity index 98% rename from app/services/impl/exam/reading/import_reading.py rename to ielts_be/services/impl/exam/reading/import_reading.py index 2888e05..1072e8b 100644 --- a/app/services/impl/exam/reading/import_reading.py +++ b/ielts_be/services/impl/exam/reading/import_reading.py @@ -5,10 +5,10 @@ from uuid import uuid4 import aiofiles from fastapi import UploadFile -from app.helpers import FileHelper -from app.mappers.reading import ReadingMapper -from app.services.abc import ILLMService -from app.dtos.exams.reading import Exam +from ielts_be.helpers import FileHelper +from ielts_be.mappers.reading import ReadingMapper +from ielts_be.services import ILLMService +from ielts_be.dtos.exams.reading import Exam class ImportReadingModule: diff --git a/app/services/impl/exam/reading/paragraph_match.py b/ielts_be/services/impl/exam/reading/paragraph_match.py similarity index 92% rename from app/services/impl/exam/reading/paragraph_match.py rename to ielts_be/services/impl/exam/reading/paragraph_match.py index b28b8da..c60bc2a 100644 --- a/app/services/impl/exam/reading/paragraph_match.py +++ b/ielts_be/services/impl/exam/reading/paragraph_match.py @@ -1,9 +1,9 @@ import random import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class ParagraphMatch: diff --git a/app/services/impl/exam/reading/write_blanks.py b/ielts_be/services/impl/exam/reading/write_blanks.py similarity index 90% rename from app/services/impl/exam/reading/write_blanks.py rename to ielts_be/services/impl/exam/reading/write_blanks.py index 9934e51..4e1456c 100644 --- a/app/services/impl/exam/reading/write_blanks.py +++ b/ielts_be/services/impl/exam/reading/write_blanks.py @@ -1,8 +1,8 @@ import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class WriteBlanks: diff --git a/app/services/impl/exam/shared/__init__.py b/ielts_be/services/impl/exam/shared/__init__.py similarity index 100% rename from app/services/impl/exam/shared/__init__.py rename to ielts_be/services/impl/exam/shared/__init__.py diff --git a/app/services/impl/exam/shared/multiple_choice.py b/ielts_be/services/impl/exam/shared/multiple_choice.py similarity index 91% rename from app/services/impl/exam/shared/multiple_choice.py rename to ielts_be/services/impl/exam/shared/multiple_choice.py index 5418331..06c33a4 100644 --- a/app/services/impl/exam/shared/multiple_choice.py +++ b/ielts_be/services/impl/exam/shared/multiple_choice.py @@ -1,8 +1,8 @@ import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class MultipleChoice: diff --git a/app/services/impl/exam/shared/true_false.py b/ielts_be/services/impl/exam/shared/true_false.py similarity index 92% rename from app/services/impl/exam/shared/true_false.py rename to ielts_be/services/impl/exam/shared/true_false.py index 903efe0..f7bea24 100644 --- a/app/services/impl/exam/shared/true_false.py +++ b/ielts_be/services/impl/exam/shared/true_false.py @@ -1,8 +1,8 @@ import uuid -from app.configs.constants import GPTModels, TemperatureSettings -from app.helpers import ExercisesHelper -from app.services.abc import ILLMService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import ExercisesHelper +from ielts_be.services import ILLMService class TrueFalse: diff --git a/ielts_be/services/impl/exam/speaking/__init__.py b/ielts_be/services/impl/exam/speaking/__init__.py new file mode 100644 index 0000000..007316b --- /dev/null +++ b/ielts_be/services/impl/exam/speaking/__init__.py @@ -0,0 +1,168 @@ +import logging +import re + +from typing import Dict, List + +from ielts_be.configs.constants import ( + FieldsAndExercises, GPTModels, TemperatureSettings +) +from ielts_be.dtos.speaking import GradeSpeakingItem +from ielts_be.repositories import IFileStorage +from ielts_be.services import ISpeakingService, ILLMService, ISpeechToTextService +from .grade import GradeSpeaking + + +class SpeakingService(ISpeakingService): + + def __init__( + self, llm: ILLMService, + file_storage: IFileStorage, + stt: ISpeechToTextService + ): + self._llm = llm + self._file_storage = file_storage + self._stt = stt + self._logger = logging.getLogger(__name__) + self._grade = GradeSpeaking(llm, file_storage, stt) + + # TODO: Is the difficulty in the prompts supposed to be hardcoded? The response is set with + # either the difficulty in the request or a random one yet the prompt doesn't change + self._tasks = { + "task_1": { + "get": { + "json_template": { + "first_topic": "topic 1", + "second_topic": "topic 2", + "questions": [ + ( + "Introductory question about the first topic, starting the topic with " + "'Let's talk about x' and then the question." + ), + "Follow up question about the first topic", + "Follow up question about the first topic", + "Question about second topic", + "Follow up question about the second topic", + ] + }, + "prompt": ( + 'Craft 5 simple and single questions of easy difficulty for IELTS Speaking Part 1 ' + 'that encourages candidates to delve deeply into personal experiences, preferences, or ' + 'insights on the topic of "{first_topic}" and the topic of "{second_topic}". ' + 'Make sure that the generated question does not contain forbidden subjects in ' + 'muslim countries.' + ) + } + }, + "task_2": { + "get": { + "json_template": { + "topic": "topic", + "question": "question", + "prompts": [ + "prompt_1", + "prompt_2", + "prompt_3" + ], + "suffix": "And explain why..." + }, + "prompt": ( + 'Create a question of medium difficulty for IELTS Speaking Part 2 ' + 'that encourages candidates to narrate a personal experience or story related to the topic ' + 'of "{topic}". Include 3 prompts that guide the candidate to describe ' + 'specific aspects of the experience, such as details about the situation, ' + 'their actions, and the reasons it left a lasting impression. Make sure that the ' + 'generated question does not contain forbidden subjects in muslim countries.' + ) + } + }, + "task_3": { + "get": { + "json_template": { + "topic": "topic", + "questions": [ + "Introductory question about the topic.", + "Follow up question about the topic", + "Follow up question about the topic", + "Follow up question about the topic", + "Follow up question about the topic" + ] + }, + "prompt": ( + 'Formulate a set of 5 single questions of hard difficulty for IELTS Speaking Part 3' + 'that encourage candidates to engage in a meaningful discussion on the topic of "{topic}". ' + 'Provide inquiries, ensuring they explore various aspects, perspectives, and implications ' + 'related to the topic. Make sure that the generated question does not contain forbidden ' + 'subjects in muslim countries.' + ) + } + }, + } + + async def get_speaking_part( + self, part: int, topic: str, second_topic: str, difficulty: str + ) -> Dict: + task_values = self._tasks[f'task_{part}']['get'] + + if part == 1: + task_prompt = task_values["prompt"].format(first_topic=topic, second_topic=second_topic) + else: + task_prompt = task_values["prompt"].format(topic=topic) + + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + f'{task_values["json_template"]}' + ) + }, + { + "role": "user", + "content": task_prompt + } + ] + + part_specific = { + "1": 'The questions should lead to the usage of 4 verb tenses (present perfect, present, past and future).', + "2": ( + 'The prompts must not be questions. Also include a suffix like the ones in the IELTS exams ' + 'that start with "And explain why".' + ) + } + + if part in {1, 2}: + messages.append({ + "role": "user", + "content": part_specific[str(part)] + }) + + if part in {1, 3}: + messages.append({ + "role": "user", + "content": 'They must be 1 single question each and not be double-barreled questions.' + }) + + fields_to_check = ["first_topic"] if part == 1 else FieldsAndExercises.GEN_FIELDS + + response = await self._llm.prediction( + GPTModels.GPT_4_O, messages, fields_to_check, TemperatureSettings.GEN_QUESTION_TEMPERATURE + ) + + if part == 3: + # Remove the numbers from the questions only if the string starts with a number + response["questions"] = [ + re.sub(r"^\d+\.\s*", "", question) + if re.match(r"^\d+\.", question) else question + for question in response["questions"] + ] + + response["type"] = part + response["difficulty"] = difficulty + + if part in {2, 3}: + response["topic"] = topic + + return response + + async def grade_speaking_task(self, task: int, items: List[GradeSpeakingItem]) -> Dict: + return await self._grade.grade_speaking_task(task, items) diff --git a/app/services/impl/exam/speaking.py b/ielts_be/services/impl/exam/speaking/grade.py similarity index 60% rename from app/services/impl/exam/speaking.py rename to ielts_be/services/impl/exam/speaking/grade.py index 62fb2cc..6d73d6c 100644 --- a/app/services/impl/exam/speaking.py +++ b/ielts_be/services/impl/exam/speaking/grade.py @@ -1,468 +1,316 @@ -import asyncio -import logging -import os -from uuid import uuid4 - -import aiofiles -import re -import uuid -from typing import Dict, List, Optional - -from app.configs.constants import ( - FieldsAndExercises, GPTModels, TemperatureSettings, - FilePaths -) -from app.dtos.speaking import GradeSpeakingItem -from app.helpers import TextHelper -from app.repositories.abc import IFileStorage, IDocumentStore -from app.services.abc import ISpeakingService, ILLMService, IVideoGeneratorService, ISpeechToTextService - - -class SpeakingService(ISpeakingService): - - def __init__( - self, llm: ILLMService, - file_storage: IFileStorage, - stt: ISpeechToTextService - ): - self._llm = llm - self._file_storage = file_storage - self._stt = stt - self._logger = logging.getLogger(__name__) - - # TODO: Is the difficulty in the prompts supposed to be hardcoded? The response is set with - # either the difficulty in the request or a random one yet the prompt doesn't change - self._tasks = { - "task_1": { - "get": { - "json_template": { - "first_topic": "topic 1", - "second_topic": "topic 2", - "questions": [ - ( - "Introductory question about the first topic, starting the topic with " - "'Let's talk about x' and then the question." - ), - "Follow up question about the first topic", - "Follow up question about the first topic", - "Question about second topic", - "Follow up question about the second topic", - ] - }, - "prompt": ( - 'Craft 5 simple and single questions of easy difficulty for IELTS Speaking Part 1 ' - 'that encourages candidates to delve deeply into personal experiences, preferences, or ' - 'insights on the topic of "{first_topic}" and the topic of "{second_topic}". ' - 'Make sure that the generated question does not contain forbidden subjects in ' - 'muslim countries.' - ) - } - }, - "task_2": { - "get": { - "json_template": { - "topic": "topic", - "question": "question", - "prompts": [ - "prompt_1", - "prompt_2", - "prompt_3" - ], - "suffix": "And explain why..." - }, - "prompt": ( - 'Create a question of medium difficulty for IELTS Speaking Part 2 ' - 'that encourages candidates to narrate a personal experience or story related to the topic ' - 'of "{topic}". Include 3 prompts that guide the candidate to describe ' - 'specific aspects of the experience, such as details about the situation, ' - 'their actions, and the reasons it left a lasting impression. Make sure that the ' - 'generated question does not contain forbidden subjects in muslim countries.' - ) - } - }, - "task_3": { - "get": { - "json_template": { - "topic": "topic", - "questions": [ - "Introductory question about the topic.", - "Follow up question about the topic", - "Follow up question about the topic", - "Follow up question about the topic", - "Follow up question about the topic" - ] - }, - "prompt": ( - 'Formulate a set of 5 single questions of hard difficulty for IELTS Speaking Part 3' - 'that encourage candidates to engage in a meaningful discussion on the topic of "{topic}". ' - 'Provide inquiries, ensuring they explore various aspects, perspectives, and implications ' - 'related to the topic. Make sure that the generated question does not contain forbidden ' - 'subjects in muslim countries.' - ) - } - }, - } - - async def get_speaking_part( - self, part: int, topic: str, second_topic: str, difficulty: str - ) -> Dict: - task_values = self._tasks[f'task_{part}']['get'] - - if part == 1: - task_prompt = task_values["prompt"].format(first_topic=topic, second_topic=second_topic) - else: - task_prompt = task_values["prompt"].format(topic=topic) - - messages = [ - { - "role": "system", - "content": ( - 'You are a helpful assistant designed to output JSON on this format: ' - f'{task_values["json_template"]}' - ) - }, - { - "role": "user", - "content": task_prompt - } - ] - - part_specific = { - "1": 'The questions should lead to the usage of 4 verb tenses (present perfect, present, past and future).', - "2": ( - 'The prompts must not be questions. Also include a suffix like the ones in the IELTS exams ' - 'that start with "And explain why".' - ) - } - - if part in {1, 2}: - messages.append({ - "role": "user", - "content": part_specific[str(part)] - }) - - if part in {1, 3}: - messages.append({ - "role": "user", - "content": 'They must be 1 single question each and not be double-barreled questions.' - }) - - fields_to_check = ["first_topic"] if part == 1 else FieldsAndExercises.GEN_FIELDS - - response = await self._llm.prediction( - GPTModels.GPT_4_O, messages, fields_to_check, TemperatureSettings.GEN_QUESTION_TEMPERATURE - ) - - if part == 3: - # Remove the numbers from the questions only if the string starts with a number - response["questions"] = [ - re.sub(r"^\d+\.\s*", "", question) - if re.match(r"^\d+\.", question) else question - for question in response["questions"] - ] - - response["type"] = part - response["difficulty"] = difficulty - - if part in {2, 3}: - response["topic"] = topic - - return response - - async def grade_speaking_task(self, task: int, items: List[GradeSpeakingItem]) -> Dict: - request_id = str(uuid.uuid4()) - self._log(task, request_id, f"Received request to grade speaking task {task}.") - - if task != 2: - self._log(task, request_id, f'Received {len(items)} total answers.') - - temp_files = [] - try: - # Save all files first - temp_files = await asyncio.gather(*[ - self.save_file(item) for item in items - ]) - - # Process all transcriptions concurrently (up to 4) - self._log(task, request_id, 'Starting batch transcription') - text_answers = await asyncio.gather(*[ - self._stt.speech_to_text(file_path) - for file_path in temp_files - ]) - - for answer in text_answers: - self._log(task, request_id, f'Transcribed answer: {answer}') - if not TextHelper.has_x_words(answer, 20): - self._log( - task, request_id, - f'The answer had less words than threshold 20 to be graded. Answer: {answer}' - ) - return self._zero_rating("The audio recorded does not contain enough english words to be graded.") - - # Get perfect answers - self._log(task, request_id, 'Requesting perfect answers') - perfect_answers = await asyncio.gather(*[ - self._get_perfect_answer(task, item.question) - for item in items - ]) - - # Format the responses - if task in {1, 3}: - self._log(task, request_id, 'Formatting answers and questions for prompt.') - - formatted_text = "" - for i, (item, transcribed_answer) in enumerate(zip(items, text_answers), start=1): - formatted_text += f"**Question {i}:**\n{item.question}\n\n" - formatted_text += f"**Answer {i}:**\n{transcribed_answer}\n\n" - - self._log(task, request_id, f'Formatted answers and questions for prompt: {formatted_text}') - questions_and_answers = f'\n\n The questions and answers are: \n\n{formatted_text}' - else: - questions_and_answers = f'\n Question: "{items[0].question}" \n Answer: "{text_answers[0]}"' - - self._log(task, request_id, 'Requesting grading of the answer(s).') - response = await self._grade_task(task, questions_and_answers) - self._log(task, request_id, f'Answer(s) graded: {response}') - - if task in {1, 3}: - self._log(task, request_id, 'Adding perfect answer(s) to response.') - - # TODO: check if it is answer["answer"] instead - for i, answer in enumerate(perfect_answers, start=1): - response['perfect_answer_' + str(i)] = answer - - self._log(task, request_id, 'Getting speaking corrections in parallel') - # Get all corrections in parallel - fixed_texts = await asyncio.gather(*[ - self._get_speaking_corrections(answer) - for answer in text_answers - ]) - - self._log(task, request_id, 'Adding transcript and fixed texts to response.') - for i, (answer, fixed) in enumerate(zip(text_answers, fixed_texts), start=1): - response['transcript_' + str(i)] = answer - response['fixed_text_' + str(i)] = fixed - else: - response['transcript'] = text_answers[0] - - self._log(task, request_id, 'Requesting fixed text.') - response['fixed_text'] = await self._get_speaking_corrections(text_answers[0]) - self._log(task, request_id, f'Fixed text: {response["fixed_text"]}') - - response['perfect_answer'] = perfect_answers[0]["answer"] - - solutions = [] - for file_name in temp_files: - solutions.append(await self._file_storage.upload_file_firebase_get_url(f'{FilePaths.FIREBASE_SPEAKING_VIDEO_FILES_PATH}{uuid4()}.wav', file_name)) - - response["overall"] = self._fix_speaking_overall(response["overall"], response["task_response"]) - response["solutions"] = solutions - if task in {1,3}: - response["answer"] = solutions - else: - response["fullPath"] = solutions[0] - - self._log(task, request_id, f'Final response: {response}') - return response - - finally: - for file_path in temp_files: - try: - if os.path.exists(file_path): - os.remove(file_path) - except Exception as e: - self._log(task, request_id, f'Error cleaning up temp file {file_path}: {str(e)}') - - def _log(self, task: int, request_id: str, message: str): - self._logger.info(f'POST - speaking_task_{task} - {request_id} - {message}') - - @staticmethod - async def save_file(item: GradeSpeakingItem) -> str: - sound_file_name = "tmp/" + str(uuid.uuid4()) - content = await item.answer.read() - async with aiofiles.open(sound_file_name, 'wb') as f: - await f.write(content) - return sound_file_name - - # ================================================================================================================== - # grade_speaking_task helpers - # ================================================================================================================== - - async def _get_perfect_answer(self, task: int, question: str): - messages = [ - { - "role": "system", - "content": ( - 'You are a helpful assistant designed to output JSON on this format: {"answer": "perfect answer"}' - ) - }, - { - "role": "user", - "content": ( - 'Provide a perfect answer according to ielts grading system to the following ' - f'Speaking Part {task} question: "{question}"' - ) - } - ] - - if task == 1: - messages.append({ - "role": "user", - "content": 'The answer must be 2 or 3 sentences long.' - }) - - gpt_model = GPTModels.GPT_4_O if task == 1 else GPTModels.GPT_3_5_TURBO - - return await self._llm.prediction( - gpt_model, messages, ["answer"], TemperatureSettings.GRADING_TEMPERATURE - ) - - async def _grade_task(self, task: int, questions_and_answers: str) -> Dict: - messages = [ - { - "role": "system", - "content": ( - f'You are a helpful assistant designed to output JSON on this format: {self._grade_template()}' - ) - }, - { - "role": "user", - "content": ( - f'Evaluate the given Speaking Part {task} response based on the IELTS grading system, ensuring a ' - 'strict assessment that penalizes errors. Deduct points for deviations from the task, and ' - 'assign a score of 0 if the response fails to address the question. Additionally, provide ' - 'detailed commentary highlighting both strengths and weaknesses in the response.' - ) + questions_and_answers - } - ] - - task_specific = { - "1": ( - 'Address the student as "you". If the answers are not 2 or 3 sentences long, warn the ' - 'student that they should be.' - ), - "2": 'Address the student as "you"', - "3": 'Address the student as "you" and pay special attention to coherence between the answers.' - } - - messages.append({ - "role": "user", - "content": task_specific[str(task)] - }) - - if task in {1, 3}: - messages.extend([ - { - "role": "user", - "content": ( - 'For pronunciations act as if you heard the answers and they were transcribed ' - 'as you heard them.' - ) - }, - { - "role": "user", - "content": 'The comments must be long, detailed, justify the grading and suggest improvements.' - } - ]) - - return await self._llm.prediction( - GPTModels.GPT_4_O, messages, ["comment"], TemperatureSettings.GRADING_TEMPERATURE - ) - - @staticmethod - def _fix_speaking_overall(overall: float, task_response: dict): - grades = [category["grade"] for category in task_response.values()] - - if overall > max(grades) or overall < min(grades): - total_sum = sum(grades) - average = total_sum / len(grades) - rounded_average = round(average, 0) - return rounded_average - - return overall - - @staticmethod - def _zero_rating(comment: str): - return { - "comment": comment, - "overall": 0, - "task_response": { - "Fluency and Coherence": { - "grade": 0.0, - "comment": "" - }, - "Lexical Resource": { - "grade": 0.0, - "comment": "" - }, - "Grammatical Range and Accuracy": { - "grade": 0.0, - "comment": "" - }, - "Pronunciation": { - "grade": 0.0, - "comment": "" - } - } - } - - async def _get_speaking_corrections(self, text): - messages = [ - { - "role": "system", - "content": ( - 'You are a helpful assistant designed to output JSON on this format: ' - '{"fixed_text": "fixed transcription with no misspelling errors"}' - ) - }, - { - "role": "user", - "content": ( - 'Fix the errors in the provided transcription and put it in a JSON. ' - f'Do not complete the answer, only replace what is wrong. \n The text: "{text}"' - ) - } - ] - - response = await self._llm.prediction( - GPTModels.GPT_3_5_TURBO, - messages, - ["fixed_text"], - 0.2, - False - ) - return response["fixed_text"] - - - @staticmethod - def _grade_template(): - return { - "comment": "extensive comment about answer quality", - "overall": 0.0, - "task_response": { - "Fluency and Coherence": { - "grade": 0.0, - "comment": ( - "extensive comment about fluency and coherence, use examples to justify the grade awarded." - ) - }, - "Lexical Resource": { - "grade": 0.0, - "comment": "extensive comment about lexical resource, use examples to justify the grade awarded." - }, - "Grammatical Range and Accuracy": { - "grade": 0.0, - "comment": ( - "extensive comment about grammatical range and accuracy, use examples to justify the " - "grade awarded." - ) - }, - "Pronunciation": { - "grade": 0.0, - "comment": ( - "extensive comment about pronunciation on the transcribed answer, use examples to justify the " - "grade awarded." - ) - } - } - } \ No newline at end of file +import asyncio +import os +import uuid +from logging import getLogger +from typing import Dict, List + +import aiofiles + +from ielts_be.configs.constants import GPTModels, TemperatureSettings, FilePaths +from ielts_be.dtos.speaking import GradeSpeakingItem +from ielts_be.helpers import TextHelper +from ielts_be.repositories import IFileStorage +from ielts_be.services import ILLMService, ISpeechToTextService + + +class GradeSpeaking: + + def __init__(self, llm: ILLMService, file_storage: IFileStorage, stt: ISpeechToTextService): + self._llm = llm + self._file_storage = file_storage + self._stt = stt + self._logger = getLogger(__name__) + + async def grade_speaking_task(self, task: int, items: List[GradeSpeakingItem]) -> Dict: + request_id = str(uuid.uuid4()) + self._log(task, request_id, f"Received request to grade speaking task {task}.") + + if task != 2: + self._log(task, request_id, f'Received {len(items)} total answers.') + + temp_files = [] + try: + # Save all files first + temp_files = await asyncio.gather(*[ + self.save_file(item) for item in items + ]) + + # Process all transcriptions concurrently (up to 4) + self._log(task, request_id, 'Starting batch transcription') + text_answers = await asyncio.gather(*[ + self._stt.speech_to_text(file_path) + for file_path in temp_files + ]) + + for answer in text_answers: + self._log(task, request_id, f'Transcribed answer: {answer}') + if not TextHelper.has_x_words(answer, 20): + self._log( + task, request_id, + f'The answer had less words than threshold 20 to be graded. Answer: {answer}' + ) + return self._zero_rating("The audio recorded does not contain enough english words to be graded.") + + # Get perfect answers + self._log(task, request_id, 'Requesting perfect answers') + perfect_answers = await asyncio.gather(*[ + self._get_perfect_answer(task, item.question) + for item in items + ]) + + # Format the responses + if task in {1, 3}: + self._log(task, request_id, 'Formatting answers and questions for prompt.') + + formatted_text = "" + for i, (item, transcribed_answer) in enumerate(zip(items, text_answers), start=1): + formatted_text += f"**Question {i}:**\n{item.question}\n\n" + formatted_text += f"**Answer {i}:**\n{transcribed_answer}\n\n" + + self._log(task, request_id, f'Formatted answers and questions for prompt: {formatted_text}') + questions_and_answers = f'\n\n The questions and answers are: \n\n{formatted_text}' + else: + questions_and_answers = f'\n Question: "{items[0].question}" \n Answer: "{text_answers[0]}"' + + self._log(task, request_id, 'Requesting grading of the answer(s).') + response = await self._grade_task(task, questions_and_answers) + self._log(task, request_id, f'Answer(s) graded: {response}') + + if task in {1, 3}: + self._log(task, request_id, 'Adding perfect answer(s) to response.') + + # TODO: check if it is answer["answer"] instead + for i, answer in enumerate(perfect_answers, start=1): + response['perfect_answer_' + str(i)] = answer + + self._log(task, request_id, 'Getting speaking corrections in parallel') + # Get all corrections in parallel + fixed_texts = await asyncio.gather(*[ + self._get_speaking_corrections(answer) + for answer in text_answers + ]) + + self._log(task, request_id, 'Adding transcript and fixed texts to response.') + for i, (answer, fixed) in enumerate(zip(text_answers, fixed_texts), start=1): + response['transcript_' + str(i)] = answer + response['fixed_text_' + str(i)] = fixed + else: + response['transcript'] = text_answers[0] + + self._log(task, request_id, 'Requesting fixed text.') + response['fixed_text'] = await self._get_speaking_corrections(text_answers[0]) + self._log(task, request_id, f'Fixed text: {response["fixed_text"]}') + + response['perfect_answer'] = perfect_answers[0]["answer"] + + solutions = [] + for file_name in temp_files: + solutions.append(await self._file_storage.upload_file_firebase_get_url(f'{FilePaths.FIREBASE_SPEAKING_VIDEO_FILES_PATH}{uuid.uuid4()}.wav', file_name)) + + response["overall"] = self._fix_speaking_overall(response["overall"], response["task_response"]) + response["solutions"] = solutions + if task in {1,3}: + response["answer"] = solutions + else: + response["fullPath"] = solutions[0] + + self._log(task, request_id, f'Final response: {response}') + return response + + finally: + for file_path in temp_files: + try: + if os.path.exists(file_path): + os.remove(file_path) + except Exception as e: + self._log(task, request_id, f'Error cleaning up temp file {file_path}: {str(e)}') + + def _log(self, task: int, request_id: str, message: str): + self._logger.info(f'POST - speaking_task_{task} - {request_id} - {message}') + + async def _get_perfect_answer(self, task: int, question: str): + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: {"answer": "perfect answer"}' + ) + }, + { + "role": "user", + "content": ( + 'Provide a perfect answer according to ielts grading system to the following ' + f'Speaking Part {task} question: "{question}"' + ) + } + ] + + if task == 1: + messages.append({ + "role": "user", + "content": 'The answer must be 2 or 3 sentences long.' + }) + + gpt_model = GPTModels.GPT_4_O if task == 1 else GPTModels.GPT_3_5_TURBO + + return await self._llm.prediction( + gpt_model, messages, ["answer"], TemperatureSettings.GRADING_TEMPERATURE + ) + + async def _grade_task(self, task: int, questions_and_answers: str) -> Dict: + messages = [ + { + "role": "system", + "content": ( + f'You are a helpful assistant designed to output JSON on this format: {self._grade_template()}' + ) + }, + { + "role": "user", + "content": ( + f'Evaluate the given Speaking Part {task} response based on the IELTS grading system, ensuring a ' + 'strict assessment that penalizes errors. Deduct points for deviations from the task, and ' + 'assign a score of 0 if the response fails to address the question. Additionally, provide ' + 'detailed commentary highlighting both strengths and weaknesses in the response.' + ) + questions_and_answers + } + ] + + task_specific = { + "1": ( + 'Address the student as "you". If the answers are not 2 or 3 sentences long, warn the ' + 'student that they should be.' + ), + "2": 'Address the student as "you"', + "3": 'Address the student as "you" and pay special attention to coherence between the answers.' + } + + messages.append({ + "role": "user", + "content": task_specific[str(task)] + }) + + if task in {1, 3}: + messages.extend([ + { + "role": "user", + "content": ( + 'For pronunciations act as if you heard the answers and they were transcribed ' + 'as you heard them.' + ) + }, + { + "role": "user", + "content": 'The comments must be long, detailed, justify the grading and suggest improvements.' + } + ]) + + return await self._llm.prediction( + GPTModels.GPT_4_O, messages, ["comment"], TemperatureSettings.GRADING_TEMPERATURE + ) + + @staticmethod + def _fix_speaking_overall(overall: float, task_response: dict): + grades = [category["grade"] for category in task_response.values()] + + if overall > max(grades) or overall < min(grades): + total_sum = sum(grades) + average = total_sum / len(grades) + rounded_average = round(average, 0) + return rounded_average + + return overall + + @staticmethod + def _zero_rating(comment: str): + return { + "comment": comment, + "overall": 0, + "task_response": { + "Fluency and Coherence": { + "grade": 0.0, + "comment": "" + }, + "Lexical Resource": { + "grade": 0.0, + "comment": "" + }, + "Grammatical Range and Accuracy": { + "grade": 0.0, + "comment": "" + }, + "Pronunciation": { + "grade": 0.0, + "comment": "" + } + } + } + + async def _get_speaking_corrections(self, text): + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + '{"fixed_text": "fixed transcription with no misspelling errors"}' + ) + }, + { + "role": "user", + "content": ( + 'Fix the errors in the provided transcription and put it in a JSON. ' + f'Do not complete the answer, only replace what is wrong. \n The text: "{text}"' + ) + } + ] + + response = await self._llm.prediction( + GPTModels.GPT_3_5_TURBO, + messages, + ["fixed_text"], + 0.2, + False + ) + return response["fixed_text"] + + + @staticmethod + def _grade_template(): + return { + "comment": "extensive comment about answer quality", + "overall": 0.0, + "task_response": { + "Fluency and Coherence": { + "grade": 0.0, + "comment": ( + "extensive comment about fluency and coherence, use examples to justify the grade awarded." + ) + }, + "Lexical Resource": { + "grade": 0.0, + "comment": "extensive comment about lexical resource, use examples to justify the grade awarded." + }, + "Grammatical Range and Accuracy": { + "grade": 0.0, + "comment": ( + "extensive comment about grammatical range and accuracy, use examples to justify the " + "grade awarded." + ) + }, + "Pronunciation": { + "grade": 0.0, + "comment": ( + "extensive comment about pronunciation on the transcribed answer, use examples to justify the " + "grade awarded." + ) + } + } + } + + @staticmethod + async def save_file(item: GradeSpeakingItem) -> str: + sound_file_name = "tmp/" + str(uuid.uuid4()) + content = await item.answer.read() + async with aiofiles.open(sound_file_name, 'wb') as f: + await f.write(content) + return sound_file_name diff --git a/ielts_be/services/impl/exam/writing/__init__.py b/ielts_be/services/impl/exam/writing/__init__.py new file mode 100644 index 0000000..d18d68d --- /dev/null +++ b/ielts_be/services/impl/exam/writing/__init__.py @@ -0,0 +1,80 @@ +from typing import List, Dict, Optional + +from fastapi import UploadFile + +from ielts_be.repositories import IFileStorage +from ielts_be.services import IWritingService, ILLMService, IAIDetectorService +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from .academic import get_writing_args_academic +from .general import get_writing_args_general +from .grade import GradeWriting + + +class WritingService(IWritingService): + + def __init__(self, llm: ILLMService, ai_detector: IAIDetectorService, file_storage: IFileStorage): + self._llm = llm + self._grade = GradeWriting(llm, file_storage, ai_detector) + + async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}' + ) + }, + *get_writing_args_general(task, topic, difficulty) + ] + + llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O + + response = await self._llm.prediction( + llm_model, + messages, + ["prompt"], + TemperatureSettings.GEN_QUESTION_TEMPERATURE + ) + + question = response["prompt"].strip() + + return { + "question": self._add_newline_before_hyphen(question) if task == 1 else question, + "difficulty": difficulty, + "topic": topic + } + + async def get_writing_task_academic_question(self, task: int, file: UploadFile, difficulty: str): + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}' + ) + }, + *(await get_writing_args_academic(task, file)) + ] + + llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O + + response = await self._llm.prediction( + llm_model, + messages, + ["prompt"], + TemperatureSettings.GEN_QUESTION_TEMPERATURE + ) + + question = response["prompt"].strip() + + return { + "question": self._add_newline_before_hyphen(question) if task == 1 else question, + "difficulty": difficulty, + } + + async def grade_writing_task(self, task: int, question: str, answer: str, attachment: Optional[str] = None): + return await self._grade.grade_writing_task(task, question, answer, attachment) + + @staticmethod + def _add_newline_before_hyphen(s): + return s.replace(" -", "\n-") + diff --git a/ielts_be/services/impl/exam/writing/academic.py b/ielts_be/services/impl/exam/writing/academic.py new file mode 100644 index 0000000..59868fc --- /dev/null +++ b/ielts_be/services/impl/exam/writing/academic.py @@ -0,0 +1,48 @@ +from base64 import b64encode +from typing import List, Dict + +from fastapi.datastructures import UploadFile + + +async def get_writing_args_academic(task: int, attachment: UploadFile) -> List[Dict]: + writing_args = { + "1": { + "prompt": ( + 'Analyze the uploaded image and create a detailed IELTS Writing Task 1 Academic prompt.\n' + 'Based on the visual data presented, craft a prompt that accurately reflects the image\'s ' + 'content, complexity, and academic nature.\n' + ), + "instructions": ( + 'The generated prompt must:\n' + '1. Clearly describe the type of visual representation in the image\n' + '2. Provide a concise context for the data shown\n' + '3. End with the standard IELTS Task 1 Academic instruction:\n' + '"Summarise the information by selecting and reporting the main features, and make comparisons where relevant."' + ) + }, + } + + if task == 2: + raise NotImplemented("Task 2 academic isn't implemented yet, current implementation still uses General Task 2 prompts.") + + messages = [ + { + "role": "user", + "content": writing_args[str(task)]["prompt"] + }, + { + "role": "user", + "content": writing_args[str(task)]["instructions"] + } + ] + + if task == 1: + attachment_bytes = await attachment.read() + messages.append({ + "type": "image_url", + "image_url": { + "url": f"data:image/{attachment.filename.split('.')[-1]};base64,{b64encode(attachment_bytes).decode('utf-8')}" + } + }) + + return messages diff --git a/ielts_be/services/impl/exam/writing/general.py b/ielts_be/services/impl/exam/writing/general.py new file mode 100644 index 0000000..12621d3 --- /dev/null +++ b/ielts_be/services/impl/exam/writing/general.py @@ -0,0 +1,44 @@ +from typing import List, Dict + + +def get_writing_args_general(task: int, topic: str, difficulty: str) -> List[Dict]: + writing_args = { + "1": { + "prompt": ( + 'Craft a prompt for an IELTS Writing Task 1 General Training exercise that instructs the ' + 'student to compose a letter. The prompt should present a specific scenario or situation, ' + f'based on the topic of "{topic}", requiring the student to provide information, ' + 'advice, or instructions within the letter. Make sure that the generated prompt is ' + f'of {difficulty} difficulty and does not contain forbidden subjects in muslim countries.' + ), + "instructions": ( + 'The prompt should end with "In the letter you should" followed by 3 bullet points of what ' + 'the answer should include.' + ) + }, + "2": { + # TODO: Should the muslim disclaimer be here as well? + "prompt": ( + f'Craft a comprehensive question of {difficulty} difficulty like the ones for IELTS ' + 'Writing Task 2 General Training that directs the candidate to delve into an in-depth ' + f'analysis of contrasting perspectives on the topic of "{topic}".' + ), + "instructions": ( + 'The question should lead to an answer with either "theories", "complicated information" or ' + 'be "very descriptive" on the topic.' + ) + } + } + + messages = [ + { + "role": "user", + "content": writing_args[str(task)]["prompt"] + }, + { + "role": "user", + "content": writing_args[str(task)]["instructions"] + } + ] + + return messages diff --git a/app/services/impl/exam/writing.py b/ielts_be/services/impl/exam/writing/grade.py similarity index 62% rename from app/services/impl/exam/writing.py rename to ielts_be/services/impl/exam/writing/grade.py index d234869..d725ec3 100644 --- a/app/services/impl/exam/writing.py +++ b/ielts_be/services/impl/exam/writing/grade.py @@ -1,268 +1,207 @@ -import asyncio -from typing import List, Dict - -from app.services.abc import IWritingService, ILLMService, IAIDetectorService -from app.configs.constants import GPTModels, TemperatureSettings, FieldsAndExercises -from app.helpers import TextHelper, ExercisesHelper - - -class WritingService(IWritingService): - - def __init__(self, llm: ILLMService, ai_detector: IAIDetectorService): - self._llm = llm - self._ai_detector = ai_detector - - async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): - messages = [ - { - "role": "system", - "content": ( - 'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}' - ) - }, - *self._get_writing_args(task, topic, difficulty) - ] - - llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O - - response = await self._llm.prediction( - llm_model, - messages, - ["prompt"], - TemperatureSettings.GEN_QUESTION_TEMPERATURE - ) - - question = response["prompt"].strip() - - return { - "question": self._add_newline_before_hyphen(question) if task == 1 else question, - "difficulty": difficulty, - "topic": topic - } - - @staticmethod - def _get_writing_args(task: int, topic: str, difficulty: str) -> List[Dict]: - writing_args = { - "1": { - "prompt": ( - 'Craft a prompt for an IELTS Writing Task 1 General Training exercise that instructs the ' - 'student to compose a letter. The prompt should present a specific scenario or situation, ' - f'based on the topic of "{topic}", requiring the student to provide information, ' - 'advice, or instructions within the letter. Make sure that the generated prompt is ' - f'of {difficulty} difficulty and does not contain forbidden subjects in muslim countries.' - ), - "instructions": ( - 'The prompt should end with "In the letter you should" followed by 3 bullet points of what ' - 'the answer should include.' - ) - }, - "2": { - # TODO: Should the muslim disclaimer be here as well? - "prompt": ( - f'Craft a comprehensive question of {difficulty} difficulty like the ones for IELTS ' - 'Writing Task 2 General Training that directs the candidate to delve into an in-depth ' - f'analysis of contrasting perspectives on the topic of "{topic}".' - ), - "instructions": ( - 'The question should lead to an answer with either "theories", "complicated information" or ' - 'be "very descriptive" on the topic.' - ) - } - } - - messages = [ - { - "role": "user", - "content": writing_args[str(task)]["prompt"] - }, - { - "role": "user", - "content": writing_args[str(task)]["instructions"] - } - ] - - return messages - - async def grade_writing_task(self, task: int, question: str, answer: str): - bare_minimum = 100 if task == 1 else 180 - - if not TextHelper.has_words(answer): - return self._zero_rating("The answer does not contain enough english words.") - elif not TextHelper.has_x_words(answer, bare_minimum): - return self._zero_rating("The answer is insufficient and too small to be graded.") - else: - template = self._get_writing_template() - messages = [ - { - "role": "system", - "content": ( - f'You are a helpful assistant designed to output JSON on this format: {template}' - ) - }, - { - "role": "user", - "content": ( - f'Evaluate the given Writing Task {task} response based on the IELTS grading system, ' - 'ensuring a strict assessment that penalizes errors. Deduct points for deviations ' - 'from the task, and assign a score of 0 if the response fails to address the question. ' - 'Additionally, provide a detailed commentary highlighting both strengths and ' - 'weaknesses in the response. ' - f'\n Question: "{question}" \n Answer: "{answer}"') - } - ] - - if task == 1: - messages.append({ - "role": "user", - "content": ( - 'Refer to the parts of the letter as: "Greeting Opener", "bullet 1", "bullet 2", ' - '"bullet 3", "closer (restate the purpose of the letter)", "closing greeting"' - ) - }) - - llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O - temperature = ( - TemperatureSettings.GRADING_TEMPERATURE - if task == 1 else - TemperatureSettings.GEN_QUESTION_TEMPERATURE - ) - - evaluation_promise = self._llm.prediction( - llm_model, - messages, - ["comment"], - temperature - ) - - perfect_answer_minimum = 150 if task == 1 else 250 - perfect_answer_promise = self._get_perfect_answer(question, perfect_answer_minimum) - fixed_text_promise = self._get_fixed_text(answer) - ai_detection_promise = self._ai_detector.run_detection(answer) - - prediction_result, perfect_answer_result, fixed_text_result, ai_detection_result = await asyncio.gather( - evaluation_promise, - perfect_answer_promise, - fixed_text_promise, - ai_detection_promise - ) - - response = prediction_result - response["perfect_answer"] = perfect_answer_result["perfect_answer"] - response["overall"] = ExercisesHelper.fix_writing_overall( - response["overall"], - response["task_response"] - ) - response['fixed_text'] = fixed_text_result - - if ai_detection_result is not None: - response['ai_detection'] = ai_detection_result - - return response - - async def _get_fixed_text(self, text): - messages = [ - { - "role": "system", - "content": ( - 'You are a helpful assistant designed to output JSON on this format: ' - '{"fixed_text": "fixed test with no misspelling errors"}' - ) - }, - { - "role": "user", - "content": ( - 'Fix the errors in the given text and put it in a JSON. ' - f'Do not complete the answer, only replace what is wrong. \n The text: "{text}"' - ) - } - ] - - response = await self._llm.prediction( - GPTModels.GPT_3_5_TURBO, - messages, - ["fixed_text"], - 0.2, - False - ) - return response["fixed_text"] - - async def _get_perfect_answer(self, question: str, size: int) -> Dict: - messages = [ - { - "role": "system", - "content": ( - 'You are a helpful assistant designed to output JSON on this format: ' - '{"perfect_answer": "perfect answer for the question"}' - ) - }, - { - "role": "user", - "content": f'Write a perfect answer for this writing exercise of a IELTS exam. Question: {question}' - - }, - { - "role": "user", - "content": f'The answer must have at least {size} words' - } - ] - return await self._llm.prediction( - GPTModels.GPT_4_O, - messages, - ["perfect_answer"], - TemperatureSettings.GEN_QUESTION_TEMPERATURE - ) - - @staticmethod - def _zero_rating(comment: str): - return { - 'comment': comment, - 'overall': 0, - 'task_response': { - 'Task Achievement': { - "grade": 0.0, - "comment": "" - }, - 'Coherence and Cohesion': { - "grade": 0.0, - "comment": "" - }, - 'Lexical Resource': { - "grade": 0.0, - "comment": "" - }, - 'Grammatical Range and Accuracy': { - "grade": 0.0, - "comment": "" - } - } - } - - @staticmethod - def _get_writing_template(): - return { - "comment": "comment about student's response quality", - "overall": 0.0, - "task_response": { - "Task Achievement": { - "grade": 0.0, - "comment": "comment about Task Achievement of the student's response" - }, - "Coherence and Cohesion": { - "grade": 0.0, - "comment": "comment about Coherence and Cohesion of the student's response" - }, - "Lexical Resource": { - "grade": 0.0, - "comment": "comment about Lexical Resource of the student's response" - }, - "Grammatical Range and Accuracy": { - "grade": 0.0, - "comment": "comment about Grammatical Range and Accuracy of the student's response" - } - } - } - - @staticmethod - def _add_newline_before_hyphen(s): - return s.replace(" -", "\n-") - +import asyncio +from typing import Dict, Optional +from uuid import uuid4 + +from ielts_be.configs.constants import GPTModels, TemperatureSettings +from ielts_be.helpers import TextHelper, ExercisesHelper, FileHelper +from ielts_be.repositories import IFileStorage +from ielts_be.services import ILLMService, IAIDetectorService + + +class GradeWriting: + + def __init__(self, llm: ILLMService, file_storage: IFileStorage, ai_detector: IAIDetectorService): + self._llm = llm + self._file_storage = file_storage + self._ai_detector = ai_detector + + async def grade_writing_task(self, task: int, question: str, answer: str, attachment: Optional[str] = None): + bare_minimum = 100 if task == 1 else 180 + + if not TextHelper.has_words(answer): + return self._zero_rating("The answer does not contain enough english words.") + elif not TextHelper.has_x_words(answer, bare_minimum): + return self._zero_rating("The answer is insufficient and too small to be graded.") + else: + template = self._get_writing_template() + messages = [ + { + "role": "system", + "content": ( + f'You are a helpful assistant designed to output JSON on this format: {template}' + ) + }, + { + "role": "user", + "content": ( + f'Evaluate the given Writing Task {task} response based on the IELTS grading system, ' + 'ensuring a strict assessment that penalizes errors. Deduct points for deviations ' + 'from the task, and assign a score of 0 if the response fails to address the question. ' + 'Additionally, provide a detailed commentary highlighting both strengths and ' + 'weaknesses in the response. ' + f'\n Question: "{question}" \n Answer: "{answer}"') + } + ] + + if task == 1: + if attachment is None: + messages.append({ + "role": "user", + "content": ( + 'Refer to the parts of the letter as: "Greeting Opener", "bullet 1", "bullet 2", ' + '"bullet 3", "closer (restate the purpose of the letter)", "closing greeting"' + ) + }) + else: + uuid = str(uuid4()) + name = attachment.split('/')[-1] + out_path = f'./tmp/{uuid}/{name}' + path = await self._file_storage.download_firebase_file(attachment, out_path) + messages.append({ + "type": "image_url", + "image_url": { + "url": f"data:image/{name.split('.')[-1]};base64,{FileHelper.encode_image(path)}" + } + }) + + llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O + temperature = ( + TemperatureSettings.GRADING_TEMPERATURE + if task == 1 else + TemperatureSettings.GEN_QUESTION_TEMPERATURE + ) + + evaluation_promise = self._llm.prediction( + llm_model, + messages, + ["comment"], + temperature + ) + + perfect_answer_minimum = 150 if task == 1 else 250 + perfect_answer_promise = self._get_perfect_answer(question, perfect_answer_minimum) + fixed_text_promise = self._get_fixed_text(answer) + ai_detection_promise = self._ai_detector.run_detection(answer) + + prediction_result, perfect_answer_result, fixed_text_result, ai_detection_result = await asyncio.gather( + evaluation_promise, + perfect_answer_promise, + fixed_text_promise, + ai_detection_promise + ) + + response = prediction_result + response["perfect_answer"] = perfect_answer_result["perfect_answer"] + response["overall"] = ExercisesHelper.fix_writing_overall( + response["overall"], + response["task_response"] + ) + response['fixed_text'] = fixed_text_result + + if ai_detection_result is not None: + response['ai_detection'] = ai_detection_result + + return response + + async def _get_fixed_text(self, text): + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + '{"fixed_text": "fixed test with no misspelling errors"}' + ) + }, + { + "role": "user", + "content": ( + 'Fix the errors in the given text and put it in a JSON. ' + f'Do not complete the answer, only replace what is wrong. \n The text: "{text}"' + ) + } + ] + + response = await self._llm.prediction( + GPTModels.GPT_3_5_TURBO, + messages, + ["fixed_text"], + 0.2, + False + ) + return response["fixed_text"] + + async def _get_perfect_answer(self, question: str, size: int) -> Dict: + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + '{"perfect_answer": "perfect answer for the question"}' + ) + }, + { + "role": "user", + "content": f'Write a perfect answer for this writing exercise of a IELTS exam. Question: {question}' + + }, + { + "role": "user", + "content": f'The answer must have at least {size} words' + } + ] + return await self._llm.prediction( + GPTModels.GPT_4_O, + messages, + ["perfect_answer"], + TemperatureSettings.GEN_QUESTION_TEMPERATURE + ) + + @staticmethod + def _zero_rating(comment: str): + return { + 'comment': comment, + 'overall': 0, + 'task_response': { + 'Task Achievement': { + "grade": 0.0, + "comment": "" + }, + 'Coherence and Cohesion': { + "grade": 0.0, + "comment": "" + }, + 'Lexical Resource': { + "grade": 0.0, + "comment": "" + }, + 'Grammatical Range and Accuracy': { + "grade": 0.0, + "comment": "" + } + } + } + + @staticmethod + def _get_writing_template(): + return { + "comment": "comment about student's response quality", + "overall": 0.0, + "task_response": { + "Task Achievement": { + "grade": 0.0, + "comment": "comment about Task Achievement of the student's response" + }, + "Coherence and Cohesion": { + "grade": 0.0, + "comment": "comment about Coherence and Cohesion of the student's response" + }, + "Lexical Resource": { + "grade": 0.0, + "comment": "comment about Lexical Resource of the student's response" + }, + "Grammatical Range and Accuracy": { + "grade": 0.0, + "comment": "comment about Grammatical Range and Accuracy of the student's response" + } + } + } diff --git a/app/services/impl/third_parties/__init__.py b/ielts_be/services/impl/third_parties/__init__.py similarity index 100% rename from app/services/impl/third_parties/__init__.py rename to ielts_be/services/impl/third_parties/__init__.py diff --git a/app/services/impl/third_parties/aws_polly.py b/ielts_be/services/impl/third_parties/aws_polly.py similarity index 91% rename from app/services/impl/third_parties/aws_polly.py rename to ielts_be/services/impl/third_parties/aws_polly.py index 36af75f..87f13ca 100644 --- a/app/services/impl/third_parties/aws_polly.py +++ b/ielts_be/services/impl/third_parties/aws_polly.py @@ -1,12 +1,10 @@ import random -from typing import Union -import aiofiles from aiobotocore.client import BaseClient -from app.dtos.listening import Dialog -from app.services.abc import ITextToSpeechService -from app.configs.constants import NeuralVoices +from ielts_be.dtos.listening import Dialog +from ielts_be.services import ITextToSpeechService +from ielts_be.configs.constants import NeuralVoices class AWSPolly(ITextToSpeechService): diff --git a/app/services/impl/third_parties/elai/__init__.py b/ielts_be/services/impl/third_parties/elai/__init__.py similarity index 96% rename from app/services/impl/third_parties/elai/__init__.py rename to ielts_be/services/impl/third_parties/elai/__init__.py index c74eac5..8c1738f 100644 --- a/app/services/impl/third_parties/elai/__init__.py +++ b/ielts_be/services/impl/third_parties/elai/__init__.py @@ -2,8 +2,8 @@ from copy import deepcopy from logging import getLogger from httpx import AsyncClient -from app.dtos.video import Task, TaskStatus -from app.services.abc import IVideoGeneratorService +from ielts_be.dtos.video import Task, TaskStatus +from ielts_be.services import IVideoGeneratorService class ELAI(IVideoGeneratorService): diff --git a/app/services/impl/third_parties/elai/avatars.json b/ielts_be/services/impl/third_parties/elai/avatars.json similarity index 100% rename from app/services/impl/third_parties/elai/avatars.json rename to ielts_be/services/impl/third_parties/elai/avatars.json diff --git a/app/services/impl/third_parties/elai/conf.json b/ielts_be/services/impl/third_parties/elai/conf.json similarity index 100% rename from app/services/impl/third_parties/elai/conf.json rename to ielts_be/services/impl/third_parties/elai/conf.json diff --git a/app/services/impl/third_parties/gpt_zero.py b/ielts_be/services/impl/third_parties/gpt_zero.py similarity index 92% rename from app/services/impl/third_parties/gpt_zero.py rename to ielts_be/services/impl/third_parties/gpt_zero.py index 051bbcf..bdd819c 100644 --- a/app/services/impl/third_parties/gpt_zero.py +++ b/ielts_be/services/impl/third_parties/gpt_zero.py @@ -3,7 +3,7 @@ from typing import Dict, Optional from httpx import AsyncClient -from app.services.abc.third_parties.ai_detector import IAIDetectorService +from ielts_be.services import IAIDetectorService class GPTZero(IAIDetectorService): diff --git a/app/services/impl/third_parties/heygen/__init__.py b/ielts_be/services/impl/third_parties/heygen/__init__.py similarity index 94% rename from app/services/impl/third_parties/heygen/__init__.py rename to ielts_be/services/impl/third_parties/heygen/__init__.py index b9fc68a..050f9d8 100644 --- a/app/services/impl/third_parties/heygen/__init__.py +++ b/ielts_be/services/impl/third_parties/heygen/__init__.py @@ -1,15 +1,10 @@ -import asyncio -import os import logging -import random from copy import deepcopy -import aiofiles - from httpx import AsyncClient -from app.dtos.video import Task, TaskStatus -from app.services.abc import IVideoGeneratorService +from ielts_be.dtos.video import Task, TaskStatus +from ielts_be.services import IVideoGeneratorService class Heygen(IVideoGeneratorService): diff --git a/app/services/impl/third_parties/heygen/avatars.json b/ielts_be/services/impl/third_parties/heygen/avatars.json similarity index 100% rename from app/services/impl/third_parties/heygen/avatars.json rename to ielts_be/services/impl/third_parties/heygen/avatars.json diff --git a/app/services/impl/third_parties/openai.py b/ielts_be/services/impl/third_parties/openai.py similarity index 94% rename from app/services/impl/third_parties/openai.py rename to ielts_be/services/impl/third_parties/openai.py index 48c0d18..036903a 100644 --- a/app/services/impl/third_parties/openai.py +++ b/ielts_be/services/impl/third_parties/openai.py @@ -6,9 +6,9 @@ from typing import List, Optional, Callable, TypeVar from openai import AsyncOpenAI from openai.types.chat import ChatCompletionMessageParam -from app.services.abc import ILLMService -from app.helpers import count_tokens -from app.configs.constants import BLACKLISTED_WORDS +from ielts_be.services.abc import ILLMService +from ielts_be.helpers import count_tokens +from ielts_be.configs.constants import BLACKLISTED_WORDS from pydantic import BaseModel T = TypeVar('T', bound=BaseModel) diff --git a/app/services/impl/third_parties/whisper.py b/ielts_be/services/impl/third_parties/whisper.py similarity index 95% rename from app/services/impl/third_parties/whisper.py rename to ielts_be/services/impl/third_parties/whisper.py index 8a7a8d4..4ef980c 100644 --- a/app/services/impl/third_parties/whisper.py +++ b/ielts_be/services/impl/third_parties/whisper.py @@ -10,7 +10,7 @@ from typing import Dict from logging import getLogger from whisper import Whisper -from app.services.abc import ISpeechToTextService +from ielts_be.services import ISpeechToTextService """ The whisper model is not thread safe, a thread pool diff --git a/app/services/impl/training/__init__.py b/ielts_be/services/impl/training/__init__.py similarity index 100% rename from app/services/impl/training/__init__.py rename to ielts_be/services/impl/training/__init__.py diff --git a/app/services/impl/training/kb.py b/ielts_be/services/impl/training/kb.py similarity index 96% rename from app/services/impl/training/kb.py rename to ielts_be/services/impl/training/kb.py index a19ce7b..77a79b4 100644 --- a/app/services/impl/training/kb.py +++ b/ielts_be/services/impl/training/kb.py @@ -6,7 +6,7 @@ from typing import Dict, List import faiss import pickle -from app.services.abc import IKnowledgeBase +from ielts_be.services import IKnowledgeBase class TrainingContentKnowledgeBase(IKnowledgeBase): diff --git a/app/services/impl/training/training.py b/ielts_be/services/impl/training/training.py similarity index 96% rename from app/services/impl/training/training.py rename to ielts_be/services/impl/training/training.py index 87fe007..a5e67d4 100644 --- a/app/services/impl/training/training.py +++ b/ielts_be/services/impl/training/training.py @@ -1,16 +1,15 @@ import re -import uuid from datetime import datetime from functools import reduce from logging import getLogger -from typing import Dict, List +from typing import Dict -from app.configs.constants import TemperatureSettings, GPTModels -from app.helpers import count_tokens -from app.repositories.abc import IDocumentStore -from app.services.abc import ILLMService, ITrainingService, IKnowledgeBase -from app.dtos.training import * +from ielts_be.configs.constants import TemperatureSettings, GPTModels +from ielts_be.helpers import count_tokens +from ielts_be.repositories import IDocumentStore +from ielts_be.services import ILLMService, ITrainingService, IKnowledgeBase +from ielts_be.dtos.training import * class TrainingService(ITrainingService): diff --git a/app/services/impl/user.py b/ielts_be/services/impl/user.py similarity index 94% rename from app/services/impl/user.py rename to ielts_be/services/impl/user.py index 3cc2f6e..fc40f48 100644 --- a/app/services/impl/user.py +++ b/ielts_be/services/impl/user.py @@ -10,10 +10,10 @@ import pandas as pd import shortuuid -from app.dtos.user_batch import BatchUsersDTO, UserDTO -from app.helpers import FileHelper -from app.repositories.abc import IDocumentStore -from app.services.abc import IUserService +from ielts_be.dtos.user_batch import BatchUsersDTO, UserDTO +from ielts_be.helpers import FileHelper +from ielts_be.repositories import IDocumentStore +from ielts_be.services import IUserService class UserService(IUserService): diff --git a/app/utils/__init__.py b/ielts_be/utils/__init__.py similarity index 100% rename from app/utils/__init__.py rename to ielts_be/utils/__init__.py diff --git a/app/utils/handle_exception.py b/ielts_be/utils/handle_exception.py similarity index 100% rename from app/utils/handle_exception.py rename to ielts_be/utils/handle_exception.py diff --git a/ielts_be/utils/image_to_b64.py b/ielts_be/utils/image_to_b64.py new file mode 100644 index 0000000..2451707 --- /dev/null +++ b/ielts_be/utils/image_to_b64.py @@ -0,0 +1,10 @@ +def _b64_vision_images(path: str): + for filename in os.listdir(path): + if filename.startswith("cv-") and filename.endswith(".png"): + cv_dict.append({ + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{_encode_image(os.path.join(path, filename))}" + } + }) + return cv_dict \ No newline at end of file diff --git a/app/utils/logger.py b/ielts_be/utils/logger.py similarity index 100% rename from app/utils/logger.py rename to ielts_be/utils/logger.py From 0222c339fe14cf1b724a0173bcc947cb6e767eb2 Mon Sep 17 00:00:00 2001 From: Carlos-Mesquita Date: Tue, 10 Dec 2024 22:28:23 +0000 Subject: [PATCH 2/4] Forgot to remove the reference b64 image method used in another project --- ielts_be/utils/image_to_b64.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 ielts_be/utils/image_to_b64.py diff --git a/ielts_be/utils/image_to_b64.py b/ielts_be/utils/image_to_b64.py deleted file mode 100644 index 2451707..0000000 --- a/ielts_be/utils/image_to_b64.py +++ /dev/null @@ -1,10 +0,0 @@ -def _b64_vision_images(path: str): - for filename in os.listdir(path): - if filename.startswith("cv-") and filename.endswith(".png"): - cv_dict.append({ - "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{_encode_image(os.path.join(path, filename))}" - } - }) - return cv_dict \ No newline at end of file From 196f9e9c3e8f1661cd1da96585d9b327789dda37 Mon Sep 17 00:00:00 2001 From: Carlos-Mesquita Date: Wed, 11 Dec 2024 15:23:00 +0000 Subject: [PATCH 3/4] ENCOA-274 and patch to the Dockerfile, in some merge the firebase tools were left out --- Dockerfile | 5 ++++ ielts_be/configs/dependency_injection.py | 2 +- ielts_be/controllers/impl/exam/writing.py | 1 - ielts_be/controllers/impl/user.py | 1 + ielts_be/dtos/writing.py | 1 - .../services/impl/exam/writing/__init__.py | 4 +-- .../services/impl/exam/writing/academic.py | 25 +++++++++++-------- ielts_be/services/impl/exam/writing/grade.py | 15 ++++++----- .../services/impl/third_parties/openai.py | 5 +++- 9 files changed, 36 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 97e2f7c..0a9e536 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,11 @@ RUN apt update && apt install -y \ librsvg2-bin \ && rm -rf /var/lib/apt/lists/* +RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y nodejs + +RUN npm install -g firebase-tools + RUN pip install --no-cache-dir -r /app/requirements.txt EXPOSE 8000 diff --git a/ielts_be/configs/dependency_injection.py b/ielts_be/configs/dependency_injection.py index 084ffe5..f7af3aa 100644 --- a/ielts_be/configs/dependency_injection.py +++ b/ielts_be/configs/dependency_injection.py @@ -94,7 +94,7 @@ class DependencyInjector: ) self._container.writing_service = providers.Factory( - WritingService, llm=self._container.llm, ai_detector=self._container.ai_detector + WritingService, llm=self._container.llm, ai_detector=self._container.ai_detector, file_storage=self._container.firebase_instance ) with open('ielts_be/services/impl/exam/level/mc_variants.json', 'r') as file: diff --git a/ielts_be/controllers/impl/exam/writing.py b/ielts_be/controllers/impl/exam/writing.py index 75bcc17..5fdf19b 100644 --- a/ielts_be/controllers/impl/exam/writing.py +++ b/ielts_be/controllers/impl/exam/writing.py @@ -15,5 +15,4 @@ class WritingController(IWritingController): async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: str): if attachment.content_type not in ['image/jpeg', 'image/png']: raise HTTPException(status_code=400, detail="Invalid file type. Only JPEG and PNG allowed.") - return await self._service.get_writing_task_academic_question(task, attachment, difficulty) diff --git a/ielts_be/controllers/impl/user.py b/ielts_be/controllers/impl/user.py index 9802e5d..52ecb93 100644 --- a/ielts_be/controllers/impl/user.py +++ b/ielts_be/controllers/impl/user.py @@ -7,6 +7,7 @@ class UserController(IUserController): def __init__(self, user_service: IUserService): self._service = user_service + print(self._service) async def batch_import(self, batch: BatchUsersDTO): return await self._service.batch_users(batch) diff --git a/ielts_be/dtos/writing.py b/ielts_be/dtos/writing.py index ba33aa1..4dc9b4d 100644 --- a/ielts_be/dtos/writing.py +++ b/ielts_be/dtos/writing.py @@ -9,5 +9,4 @@ class WritingGradeTaskDTO(BaseModel): exerciseId: str question: str answer: str - type: str attachment: Optional[str] diff --git a/ielts_be/services/impl/exam/writing/__init__.py b/ielts_be/services/impl/exam/writing/__init__.py index d18d68d..9c9691c 100644 --- a/ielts_be/services/impl/exam/writing/__init__.py +++ b/ielts_be/services/impl/exam/writing/__init__.py @@ -55,10 +55,8 @@ class WritingService(IWritingService): *(await get_writing_args_academic(task, file)) ] - llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O - response = await self._llm.prediction( - llm_model, + GPTModels.GPT_4_O, messages, ["prompt"], TemperatureSettings.GEN_QUESTION_TEMPERATURE diff --git a/ielts_be/services/impl/exam/writing/academic.py b/ielts_be/services/impl/exam/writing/academic.py index 59868fc..4e8081f 100644 --- a/ielts_be/services/impl/exam/writing/academic.py +++ b/ielts_be/services/impl/exam/writing/academic.py @@ -25,6 +25,9 @@ async def get_writing_args_academic(task: int, attachment: UploadFile) -> List[D if task == 2: raise NotImplemented("Task 2 academic isn't implemented yet, current implementation still uses General Task 2 prompts.") + attachment_bytes = await attachment.read() + + messages = [ { "role": "user", @@ -32,17 +35,19 @@ async def get_writing_args_academic(task: int, attachment: UploadFile) -> List[D }, { "role": "user", - "content": writing_args[str(task)]["instructions"] + "content": [ + { + "type": "text", + "text": writing_args[str(task)]["instructions"], + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/{attachment.filename.split('.')[-1]};base64,{b64encode(attachment_bytes).decode('utf-8')}" + } + } + ] } ] - if task == 1: - attachment_bytes = await attachment.read() - messages.append({ - "type": "image_url", - "image_url": { - "url": f"data:image/{attachment.filename.split('.')[-1]};base64,{b64encode(attachment_bytes).decode('utf-8')}" - } - }) - return messages diff --git a/ielts_be/services/impl/exam/writing/grade.py b/ielts_be/services/impl/exam/writing/grade.py index d725ec3..1b22a30 100644 --- a/ielts_be/services/impl/exam/writing/grade.py +++ b/ielts_be/services/impl/exam/writing/grade.py @@ -57,14 +57,17 @@ class GradeWriting: name = attachment.split('/')[-1] out_path = f'./tmp/{uuid}/{name}' path = await self._file_storage.download_firebase_file(attachment, out_path) - messages.append({ - "type": "image_url", - "image_url": { - "url": f"data:image/{name.split('.')[-1]};base64,{FileHelper.encode_image(path)}" + messages.append( + { + "role": "user", + "content": { + "type": "image_url", + "image_url": { + "url": f"data:image/{name.split('.')[-1]};base64,{FileHelper.encode_image(path)}" + } } }) - llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O temperature = ( TemperatureSettings.GRADING_TEMPERATURE if task == 1 else @@ -72,7 +75,7 @@ class GradeWriting: ) evaluation_promise = self._llm.prediction( - llm_model, + GPTModels.GPT_4_O, messages, ["comment"], temperature diff --git a/ielts_be/services/impl/third_parties/openai.py b/ielts_be/services/impl/third_parties/openai.py index 036903a..95aaf1f 100644 --- a/ielts_be/services/impl/third_parties/openai.py +++ b/ielts_be/services/impl/third_parties/openai.py @@ -93,7 +93,10 @@ class OpenAI(ILLMService): def _count_total_tokens(messages): total_tokens = 0 for message in messages: - total_tokens += count_tokens(message["content"])["n_tokens"] + # Skip when content isn't text + message_content = message.get("content", None) + if message_content is not None and isinstance(message_content, str): + total_tokens += count_tokens(message["content"])["n_tokens"] return total_tokens @staticmethod From fa028aa0e769a75b945a7732a1e0b248907f2da8 Mon Sep 17 00:00:00 2001 From: Carlos-Mesquita Date: Wed, 11 Dec 2024 15:31:35 +0000 Subject: [PATCH 4/4] Forgot a print --- ielts_be/controllers/impl/user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ielts_be/controllers/impl/user.py b/ielts_be/controllers/impl/user.py index 52ecb93..9802e5d 100644 --- a/ielts_be/controllers/impl/user.py +++ b/ielts_be/controllers/impl/user.py @@ -7,7 +7,6 @@ class UserController(IUserController): def __init__(self, user_service: IUserService): self._service = user_service - print(self._service) async def batch_import(self, batch: BatchUsersDTO): return await self._service.batch_users(batch)