devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fastapi refactor update

Browse Source
This commit is contained in:
Carlos-Mesquita
2024-10-01 19:31:01 +01:00
parent f92a803d96
commit 2a032c5aba
132 changed files with 22856 additions and 10309 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.dockerignore
View File

@@ -1,8 +1,8 @@
Dockerfile Dockerfile
README.md README.md
*.pyc *.pyc
*.pyo *.pyo
*.pyd *.pyd
__pycache__ __pycache__
.pytest_cache .pytest_cache
postman postman

38
.env
View File

@@ -1,8 +1,30 @@
ENV=local OPENAI_API_KEY=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
OPENAI_API_KEY=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 JWT_SECRET_KEY=6e9c124ba92e8814719dcb0f21200c8aa4d0f119a994ac5e06eb90a366c83ab2
JWT_SECRET_KEY=6e9c124ba92e8814719dcb0f21200c8aa4d0f119a994ac5e06eb90a366c83ab2 JWT_TEST_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0In0.Emrs2D3BmMP4b3zMjw0fJTPeyMwWEBDbxx2vvaWguO0
JWT_TEST_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0In0.Emrs2D3BmMP4b3zMjw0fJTPeyMwWEBDbxx2vvaWguO0 HEY_GEN_TOKEN=MjY4MDE0MjdjZmNhNDFmYTlhZGRkNmI3MGFlMzYwZDItMTY5NTExNzY3MA==
GOOGLE_APPLICATION_CREDENTIALS=firebase-configs/encoach-staging.json GPT_ZERO_API_KEY=0195b9bb24c5439899f71230809c74af
HEY_GEN_TOKEN=MjY4MDE0MjdjZmNhNDFmYTlhZGRkNmI3MGFlMzYwZDItMTY5NTExNzY3MA== MONGODB_URI=mongodb+srv://user:JKpFBymv0WLv3STj@encoach.lz18a.mongodb.net/?retryWrites=true&w=majority&appName=EnCoach
GOOGLE_APPLICATION_CREDENTIALS=firebase-configs/encoach-staging.json
GPT_ZERO_API_KEY=0195b9bb24c5439899f71230809c74af
# Staging
ENV=staging
#
#FIREBASE_SCRYPT_B64_SIGNER_KEY="qjo/b5U5oNxA8o+PHFMZx/ZfG8ZQ7688zYmwMOcfZvVjOM6aHe4Jf270xgyrVArqLIQwFi7VkFnbysBjueMbVw=="
#FIREBASE_SCRYPT_B64_SALT_SEPARATOR="Bw=="
#FIREBASE_SCRYPT_ROUNDS=8
#FIREBASE_SCRYPT_MEM_COST=14
#FIREBASE_PROJECT_ID=encoach-staging
#MONGODB_DB=staging
# Prod
#ENV=production
#GOOGLE_APPLICATION_CREDENTIALS=firebase-configs/storied-phalanx-349916.json
#FIREBASE_SCRYPT_B64_SIGNER_KEY="vbO3Xii2lajSeSkCstq3s/dCwpXP7J2YN9rP/KRreU2vGOT1fg+wzSuy1kIhBECqJHG82tmwAilSxLFFtNKVMA=="
#FIREBASE_SCRYPT_B64_SALT_SEPARATOR="Bw=="
#FIREBASE_SCRYPT_ROUNDS=8
#FIREBASE_SCRYPT_MEM_COST=14
#FIREBASE_PROJECT_ID=storied-phalanx-349916
MONGODB_DB=staging

12
.gitignore vendored
View File

@@ -1,6 +1,6 @@
__pycache__ __pycache__
.idea .idea
.env .env
.DS_Store .DS_Store
.venv .venv
scripts _scripts

16
.idea/.gitignore generated vendored
View File

@@ -1,8 +1,8 @@
# Default ignored files # Default ignored files
/shelf/ /shelf/
/workspace.xml /workspace.xml
# Editor-based HTTP Client requests # Editor-based HTTP Client requests
/httpRequests/ /httpRequests/
# Datasource local storage ignored files # Datasource local storage ignored files
/dataSources/ /dataSources/
/dataSources.local.xml /dataSources.local.xml

49
.idea/ielts-be.iml generated
View File

@@ -1,25 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4"> <module type="PYTHON_MODULE" version="4">
<component name="Flask"> <component name="Flask">
<option name="enabled" value="true" /> <option name="enabled" value="true" />
</component> </component>
<component name="NewModuleRootManager"> <component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$"> <content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.venv" /> <excludeFolder url="file://$MODULE_DIR$/.venv" />
<excludeFolder url="file://$MODULE_DIR$/venv" /> <excludeFolder url="file://$MODULE_DIR$/venv" />
</content> <excludeFolder url="file://$MODULE_DIR$/_scripts" />
<orderEntry type="inheritedJdk" /> </content>
<orderEntry type="sourceFolder" forTests="false" /> <orderEntry type="jdk" jdkName="Python 3.11 (ielts-be)" jdkType="Python SDK" />
</component> <orderEntry type="sourceFolder" forTests="false" />
<component name="PackageRequirementsSettings"> </component>
<option name="versionSpecifier" value="Don't specify version" /> <component name="PackageRequirementsSettings">
</component> <option name="versionSpecifier" value="Don't specify version" />
<component name="TemplatesService"> </component>
<option name="TEMPLATE_CONFIGURATION" value="Jinja2" /> <component name="TemplatesService">
<option name="TEMPLATE_FOLDERS"> <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
<list> <option name="TEMPLATE_FOLDERS">
<option value="$MODULE_DIR$/../flaskProject\templates" /> <list>
</list> <option value="$MODULE_DIR$/../flaskProject\templates" />
</option> </list>
</component> </option>
</component>
</module> </module>

18
.idea/misc.xml generated
View File

@@ -1,10 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="Black"> <component name="Black">
<option name="sdkName" value="Python 3.11 (ielts-be)" /> <option name="sdkName" value="Python 3.11 (ielts-be)" />
</component> </component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (ielts-be)" project-jdk-type="Python SDK" /> <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (ielts-be)" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser"> <component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" /> <option name="shown" value="true" />
</component> </component>
</project> </project>

14
.idea/modules.xml generated
View File

@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ProjectModuleManager"> <component name="ProjectModuleManager">
<modules> <modules>
<module fileurl="file://$PROJECT_DIR$/.idea/ielts-be.iml" filepath="$PROJECT_DIR$/.idea/ielts-be.iml" /> <module fileurl="file://$PROJECT_DIR$/.idea/ielts-be.iml" filepath="$PROJECT_DIR$/.idea/ielts-be.iml" />
</modules> </modules>
</component> </component>
</project> </project>

10
.idea/vcs.xml generated
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="VcsDirectoryMappings"> <component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" /> <mapping directory="$PROJECT_DIR$" vcs="Git" />
</component> </component>
</project> </project>

82
Dockerfile
View File

@@ -1,41 +1,41 @@
FROM python:3.11-slim as requirements-stage FROM python:3.11-slim as requirements-stage
WORKDIR /tmp WORKDIR /tmp
RUN pip install poetry RUN pip install poetry
COPY pyproject.toml ./poetry.lock* /tmp/ COPY pyproject.toml ./poetry.lock* /tmp/
RUN poetry export -f requirements.txt --output requirements.txt --without-hashes RUN poetry export -f requirements.txt --output requirements.txt --without-hashes
FROM python:3.11-slim FROM python:3.11-slim
# Allow statements and log messages to immediately appear in the logs # Allow statements and log messages to immediately appear in the logs
ENV PYTHONUNBUFFERED True ENV PYTHONUNBUFFERED True
# Copy local code to the container image. # Copy local code to the container image.
ENV APP_HOME /app ENV APP_HOME /app
WORKDIR $APP_HOME WORKDIR $APP_HOME
COPY . ./ COPY . ./
COPY --from=requirements-stage /tmp/requirements.txt /app/requirements.txt COPY --from=requirements-stage /tmp/requirements.txt /app/requirements.txt
RUN apt update && apt install -y \ RUN apt update && apt install -y \
ffmpeg \ ffmpeg \
poppler-utils \ poppler-utils \
texlive-latex-base \ texlive-latex-base \
texlive-fonts-recommended \ texlive-fonts-recommended \
texlive-latex-extra \ texlive-latex-extra \
texlive-xetex \ texlive-xetex \
pandoc \ pandoc \
librsvg2-bin \ librsvg2-bin \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir -r /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt
EXPOSE 8000 EXPOSE 8000
# Run the web service on container startup. Here we use the gunicorn # Run the web service on container startup. Here we use the gunicorn
# webserver, with one worker process and 8 threads. # webserver, with one worker process and 8 threads.
# For environments with multiple CPU cores, increase the number of workers # For environments with multiple CPU cores, increase the number of workers
# to be equal to the cores available. # to be equal to the cores available.
# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. # Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling.
CMD exec uvicorn --bind 0.0.0.0:8000 --workers 1 --threads 8 --timeout 0 app.server:app CMD exec uvicorn --bind 0.0.0.0:8000 --workers 1 --threads 8 --timeout 0 app.server:app

101
README.md
View File

@@ -1,52 +1,49 @@
Latest refactor from develop's branch commit 5d5cd21 2024-08-28 Latest refactor from develop's branch commit 5d5cd21 2024-08-28
# Endpoints # Endpoints
In ielts-ui I've added a wrapper to every backend request in '/src/utils/translate.backend.endpoints.ts' to use the
new endpoints if the "BACKEND_TYPE" environment variable is set to "async", if the env variable is not present or | Method | ielts-be | This one |
with another value, the wrapper will return the old endpoint. |--------|--------------------------------------|---------------------------------------------|
| GET | /healthcheck | /api/healthcheck |
| Method | ielts-be | This one | | GET | /listening_section_1 | /api/listening/section/1 |
|--------|--------------------------------------|---------------------------------------------| | GET | /listening_section_2 | /api/listening/section/2 |
| GET | /healthcheck | /api/healthcheck | | GET | /listening_section_3 | /api/listening/section/3 |
| GET | /listening_section_1 | /api/listening/section/1 | | GET | /listening_section_4 | /api/listening/section/4 |
| GET | /listening_section_2 | /api/listening/section/2 | | POST | /listening | /api/listening |
| GET | /listening_section_3 | /api/listening/section/3 | | POST | /writing_task1 | /api/grade/writing/1 |
| GET | /listening_section_4 | /api/listening/section/4 | | POST | /writing_task2 | /api/grade/writing/2 |
| POST | /listening | /api/listening | | GET | /writing_task1_general | /api/writing/1 |
| POST | /writing_task1 | /api/grade/writing/1 | | GET | /writing_task2_general | /api/writing/2 |
| POST | /writing_task2 | /api/grade/writing/2 | | POST | /speaking_task_1 | /api/grade/speaking/1 |
| GET | /writing_task1_general | /api/writing/1 | | POST | /speaking_task_2 | /api/grade/speaking/2 |
| GET | /writing_task2_general | /api/writing/2 | | POST | /speaking_task_3 | /api/grade/speaking/3 |
| POST | /speaking_task_1 | /api/grade/speaking/1 | | GET | /speaking_task_1 | /api/speaking/1 |
| POST | /speaking_task_2 | /api/grade/speaking/2 | | GET | /speaking_task_2 | /api/speaking/2 |
| POST | /speaking_task_3 | /api/grade/speaking/3 | | GET | /speaking_task_3 | /api/speaking/3 |
| GET | /speaking_task_1 | /api/speaking/1 | | POST | /speaking | /api/speaking |
| GET | /speaking_task_2 | /api/speaking/2 | | POST | /speaking/generate_speaking_video | /api/speaking/generate_speaking_video |
| GET | /speaking_task_3 | /api/speaking/3 | | POST | /speaking/generate_interactive_video | /api/speaking/generate_interactive_video |
| POST | /speaking | /api/speaking | | GET | /reading_passage_1 | /api/reading/passage/1 |
| POST | /speaking/generate_speaking_video | /api/speaking/generate_speaking_video | | GET | /reading_passage_2 | /api/reading/passage/2 |
| POST | /speaking/generate_interactive_video | /api/speaking/generate_interactive_video | | GET | /reading_passage_3 | /api/reading/passage/3 |
| GET | /reading_passage_1 | /api/reading/passage/1 | | GET | /level | /api/level |
| GET | /reading_passage_2 | /api/reading/passage/2 | | GET | /level_utas | /api/level/utas |
| GET | /reading_passage_3 | /api/reading/passage/3 | | POST | /fetch_tips | /api/training/tips |
| GET | /level | /api/level | | POST | /grading_summary | /api/grade/summary |
| GET | /level_utas | /api/level/utas | | POST | /grade_short_answers | /api/grade/short_answers |
| POST | /fetch_tips | /api/training/tips | | POST | /upload_level | /api/level/upload |
| POST | /grading_summary | /api/grade/summary | | POST | /training_content | /api/training/ |
| POST | /grade_short_answers | /api/grade/short_answers | | POST | /custom_level | /api/level/custom |
| POST | /upload_level | /api/level/upload |
| POST | /training_content | /api/training/ | # Run the app
| POST | /custom_level | /api/level/custom |
This is for Windows, creating venv and activating it may differ based on your OS
# Run the app
1. python -m venv env
This is for Windows, creating venv and activating it may differ based on your OS 2. env\Scripts\activate
3. pip install poetry
1. python -m venv env 4. poetry install
2. env\Scripts\activate 5. python app.py
3. pip install poetry
4. poetry install
5. python app.py

55
app.py
View File

@@ -1,30 +1,25 @@
import os import click
import uvicorn
import click from dotenv import load_dotenv
import uvicorn
from dotenv import load_dotenv load_dotenv()
@click.command() @click.command()
@click.option( @click.option(
"--env", "--env",
type=click.Choice(["local", "dev", "prod"], case_sensitive=False), type=click.Choice(["local", "staging", "production"], case_sensitive=False),
default="local", default="staging",
) )
def main(env: str): def main(env: str):
load_dotenv() uvicorn.run(
os.environ["ENV"] = env app="app.server:app",
if env == "prod": host="localhost",
raise Exception("Production environment not supported yet!") port=8000,
reload=True if env != "production" else False,
uvicorn.run( workers=1,
app="app.server:app", )
host="localhost",
port=8000,
reload=True if env != "prod" else False, if __name__ == "__main__":
workers=1, main()
)
if __name__ == "__main__":
main()

38
app/api/__init__.py
View File

@@ -1,18 +1,20 @@
from fastapi import APIRouter from fastapi import APIRouter
from .home import home_router from .home import home_router
from .listening import listening_router from .listening import listening_router
from .reading import reading_router from .reading import reading_router
from .speaking import speaking_router from .speaking import speaking_router
from .training import training_router from .training import training_router
from .writing import writing_router from .writing import writing_router
from .grade import grade_router from .grade import grade_router
from .user import user_router
router = APIRouter()
router.include_router(home_router, prefix="/api", tags=["Home"]) router = APIRouter()
router.include_router(listening_router, prefix="/api/listening", tags=["Listening"]) router.include_router(home_router, prefix="/api", tags=["Home"])
router.include_router(reading_router, prefix="/api/reading", tags=["Reading"]) router.include_router(listening_router, prefix="/api/listening", tags=["Listening"])
router.include_router(speaking_router, prefix="/api/speaking", tags=["Speaking"]) router.include_router(reading_router, prefix="/api/reading", tags=["Reading"])
router.include_router(writing_router, prefix="/api/writing", tags=["Writing"]) router.include_router(speaking_router, prefix="/api/speaking", tags=["Speaking"])
router.include_router(grade_router, prefix="/api/grade", tags=["Grade"]) router.include_router(writing_router, prefix="/api/writing", tags=["Writing"])
router.include_router(training_router, prefix="/api/training", tags=["Training"]) router.include_router(grade_router, prefix="/api/grade", tags=["Grade"])
router.include_router(training_router, prefix="/api/training", tags=["Training"])
router.include_router(user_router, prefix="/api/user", tags=["Users"])

148
app/api/grade.py
View File

@@ -1,74 +1,74 @@
from dependency_injector.wiring import inject, Provide from dependency_injector.wiring import inject, Provide
from fastapi import APIRouter, Depends, Path, Request from fastapi import APIRouter, Depends, Path, Request
from app.controllers.abc import IGradeController from app.controllers.abc import IGradeController
from app.dtos.writing import WritingGradeTaskDTO from app.dtos.writing import WritingGradeTaskDTO
from app.dtos.speaking import GradeSpeakingAnswersDTO, GradeSpeakingDTO from app.dtos.speaking import GradeSpeakingAnswersDTO, GradeSpeakingDTO
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
controller = "grade_controller" controller = "grade_controller"
grade_router = APIRouter() grade_router = APIRouter()
@grade_router.post( @grade_router.post(
'/writing/{task}', '/writing/{task}',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def grade_writing_task( async def grade_writing_task(
data: WritingGradeTaskDTO, data: WritingGradeTaskDTO,
task: int = Path(..., ge=1, le=2), task: int = Path(..., ge=1, le=2),
grade_controller: IGradeController = Depends(Provide[controller]) grade_controller: IGradeController = Depends(Provide[controller])
): ):
return await grade_controller.grade_writing_task(task, data) return await grade_controller.grade_writing_task(task, data)
@grade_router.post( @grade_router.post(
'/speaking/2', '/speaking/2',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def grade_speaking_task_2( async def grade_speaking_task_2(
data: GradeSpeakingDTO, data: GradeSpeakingDTO,
grade_controller: IGradeController = Depends(Provide[controller]) grade_controller: IGradeController = Depends(Provide[controller])
): ):
return await grade_controller.grade_speaking_task(2, [data.dict()]) return await grade_controller.grade_speaking_task(2, [data.dict()])
@grade_router.post( @grade_router.post(
'/speaking/{task}', '/speaking/{task}',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def grade_speaking_task_1_and_3( async def grade_speaking_task_1_and_3(
data: GradeSpeakingAnswersDTO, data: GradeSpeakingAnswersDTO,
task: int = Path(..., ge=1, le=3), task: int = Path(..., ge=1, le=3),
grade_controller: IGradeController = Depends(Provide[controller]) grade_controller: IGradeController = Depends(Provide[controller])
): ):
return await grade_controller.grade_speaking_task(task, data.answers) return await grade_controller.grade_speaking_task(task, data.answers)
@grade_router.post( @grade_router.post(
'/summary', '/summary',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def grading_summary( async def grading_summary(
request: Request, request: Request,
grade_controller: IGradeController = Depends(Provide[controller]) grade_controller: IGradeController = Depends(Provide[controller])
): ):
data = await request.json() data = await request.json()
return await grade_controller.grading_summary(data) return await grade_controller.grading_summary(data)
@grade_router.post( @grade_router.post(
'/short_answers', '/short_answers',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def grade_short_answers( async def grade_short_answers(
request: Request, request: Request,
grade_controller: IGradeController = Depends(Provide[controller]) grade_controller: IGradeController = Depends(Provide[controller])
): ):
data = await request.json() data = await request.json()
return await grade_controller.grade_short_answers(data) return await grade_controller.grade_short_answers(data)

18
app/api/home.py
View File

@@ -1,9 +1,9 @@
from fastapi import APIRouter from fastapi import APIRouter
home_router = APIRouter() home_router = APIRouter()
@home_router.get( @home_router.get(
'/healthcheck' '/healthcheck'
) )
async def healthcheck(): async def healthcheck():
return {"healthy": True} return {"healthy": True}

110
app/api/level.py
View File

@@ -1,55 +1,55 @@
from dependency_injector.wiring import Provide, inject from dependency_injector.wiring import Provide, inject
from fastapi import APIRouter, Depends, UploadFile, Request from fastapi import APIRouter, Depends, UploadFile, Request
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
from app.controllers.abc import ILevelController from app.controllers.abc import ILevelController
controller = "level_controller" controller = "level_controller"
level_router = APIRouter() level_router = APIRouter()
@level_router.get( @level_router.get(
'/', '/',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def get_level_exam( async def get_level_exam(
level_controller: ILevelController = Depends(Provide[controller]) level_controller: ILevelController = Depends(Provide[controller])
): ):
return await level_controller.get_level_exam() return await level_controller.get_level_exam()
@level_router.get( @level_router.get(
'/utas', '/utas',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def get_level_utas( async def get_level_utas(
level_controller: ILevelController = Depends(Provide[controller]) level_controller: ILevelController = Depends(Provide[controller])
): ):
return await level_controller.get_level_utas() return await level_controller.get_level_utas()
@level_router.post( @level_router.post(
'/upload', '/upload',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def upload( async def upload(
file: UploadFile, file: UploadFile,
level_controller: ILevelController = Depends(Provide[controller]) level_controller: ILevelController = Depends(Provide[controller])
): ):
return await level_controller.upload_level(file) return await level_controller.upload_level(file)
@level_router.post( @level_router.post(
'/custom', '/custom',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def custom_level( async def custom_level(
request: Request, request: Request,
level_controller: ILevelController = Depends(Provide[controller]) level_controller: ILevelController = Depends(Provide[controller])
): ):
data = await request.json() data = await request.json()
return await level_controller.get_custom_level(data) return await level_controller.get_custom_level(data)

80
app/api/listening.py
View File

@@ -1,40 +1,40 @@
import random import random
from dependency_injector.wiring import Provide, inject from dependency_injector.wiring import Provide, inject
from fastapi import APIRouter, Depends, Path from fastapi import APIRouter, Depends, Path
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
from app.controllers.abc import IListeningController from app.controllers.abc import IListeningController
from app.configs.constants import EducationalContent from app.configs.constants import EducationalContent
from app.dtos.listening import SaveListeningDTO from app.dtos.listening import SaveListeningDTO
controller = "listening_controller" controller = "listening_controller"
listening_router = APIRouter() listening_router = APIRouter()
@listening_router.get( @listening_router.get(
'/section/{section}', '/section/{section}',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def get_listening_question( async def get_listening_question(
exercises: list[str], exercises: list[str],
section: int = Path(..., ge=1, le=4), section: int = Path(..., ge=1, le=4),
topic: str | None = None, topic: str | None = None,
difficulty: str = random.choice(EducationalContent.DIFFICULTIES), difficulty: str = random.choice(EducationalContent.DIFFICULTIES),
listening_controller: IListeningController = Depends(Provide[controller]) listening_controller: IListeningController = Depends(Provide[controller])
): ):
return await listening_controller.get_listening_question(section, topic, exercises, difficulty) return await listening_controller.get_listening_question(section, topic, exercises, difficulty)
@listening_router.post( @listening_router.post(
'/', '/',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def save_listening( async def save_listening(
data: SaveListeningDTO, data: SaveListeningDTO,
listening_controller: IListeningController = Depends(Provide[controller]) listening_controller: IListeningController = Depends(Provide[controller])
): ):
return await listening_controller.save_listening(data) return await listening_controller.save_listening(data)

56
app/api/reading.py
View File

@@ -1,28 +1,28 @@
import random import random
from dependency_injector.wiring import Provide, inject from dependency_injector.wiring import Provide, inject
from fastapi import APIRouter, Depends, Path, Query from fastapi import APIRouter, Depends, Path, Query
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
from app.configs.constants import EducationalContent from app.configs.constants import EducationalContent
from app.controllers.abc import IReadingController from app.controllers.abc import IReadingController
controller = "reading_controller" controller = "reading_controller"
reading_router = APIRouter() reading_router = APIRouter()
@reading_router.get( @reading_router.get(
'/passage/{passage}', '/passage/{passage}',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def get_reading_passage( async def get_reading_passage(
passage: int = Path(..., ge=1, le=3), passage: int = Path(..., ge=1, le=3),
topic: str = Query(default=random.choice(EducationalContent.TOPICS)), topic: str = Query(default=random.choice(EducationalContent.TOPICS)),
exercises: list[str] = Query(default=[]), exercises: list[str] = Query(default=[]),
difficulty: str = Query(default=random.choice(EducationalContent.DIFFICULTIES)), difficulty: str = Query(default=random.choice(EducationalContent.DIFFICULTIES)),
reading_controller: IReadingController = Depends(Provide[controller]) reading_controller: IReadingController = Depends(Provide[controller])
): ):
return await reading_controller.get_reading_passage(passage, topic, exercises, difficulty) return await reading_controller.get_reading_passage(passage, topic, exercises, difficulty)

194
app/api/speaking.py
View File

@@ -1,97 +1,97 @@
import random import random
from dependency_injector.wiring import inject, Provide from dependency_injector.wiring import inject, Provide
from fastapi import APIRouter, Path, Query, Depends, BackgroundTasks from fastapi import APIRouter, Path, Query, Depends, BackgroundTasks
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
from app.configs.constants import EducationalContent from app.configs.constants import EducationalContent
from app.controllers.abc import ISpeakingController from app.controllers.abc import ISpeakingController
from app.dtos.speaking import ( from app.dtos.speaking import (
SaveSpeakingDTO, GenerateVideo1DTO, GenerateVideo2DTO, GenerateVideo3DTO SaveSpeakingDTO, GenerateVideo1DTO, GenerateVideo2DTO, GenerateVideo3DTO
) )
controller = "speaking_controller" controller = "speaking_controller"
speaking_router = APIRouter() speaking_router = APIRouter()
@speaking_router.get( @speaking_router.get(
'/1', '/1',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def get_speaking_task( async def get_speaking_task(
first_topic: str = Query(default=random.choice(EducationalContent.MTI_TOPICS)), first_topic: str = Query(default=random.choice(EducationalContent.MTI_TOPICS)),
second_topic: str = Query(default=random.choice(EducationalContent.MTI_TOPICS)), second_topic: str = Query(default=random.choice(EducationalContent.MTI_TOPICS)),
difficulty: str = Query(default=random.choice(EducationalContent.DIFFICULTIES)), difficulty: str = Query(default=random.choice(EducationalContent.DIFFICULTIES)),
speaking_controller: ISpeakingController = Depends(Provide[controller]) speaking_controller: ISpeakingController = Depends(Provide[controller])
): ):
return await speaking_controller.get_speaking_part(1, first_topic, difficulty, second_topic) return await speaking_controller.get_speaking_part(1, first_topic, difficulty, second_topic)
@speaking_router.get( @speaking_router.get(
'/{task}', '/{task}',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def get_speaking_task( async def get_speaking_task(
task: int = Path(..., ge=2, le=3), task: int = Path(..., ge=2, le=3),
topic: str = Query(default=random.choice(EducationalContent.MTI_TOPICS)), topic: str = Query(default=random.choice(EducationalContent.MTI_TOPICS)),
difficulty: str = Query(default=random.choice(EducationalContent.DIFFICULTIES)), difficulty: str = Query(default=random.choice(EducationalContent.DIFFICULTIES)),
speaking_controller: ISpeakingController = Depends(Provide[controller]) speaking_controller: ISpeakingController = Depends(Provide[controller])
): ):
return await speaking_controller.get_speaking_part(task, topic, difficulty) return await speaking_controller.get_speaking_part(task, topic, difficulty)
@speaking_router.post( @speaking_router.post(
'/', '/',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def save_speaking( async def save_speaking(
data: SaveSpeakingDTO, data: SaveSpeakingDTO,
background_tasks: BackgroundTasks, background_tasks: BackgroundTasks,
speaking_controller: ISpeakingController = Depends(Provide[controller]) speaking_controller: ISpeakingController = Depends(Provide[controller])
): ):
return await speaking_controller.save_speaking(data, background_tasks) return await speaking_controller.save_speaking(data, background_tasks)
@speaking_router.post( @speaking_router.post(
'/generate_video/1', '/generate_video/1',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def generate_video_1( async def generate_video_1(
data: GenerateVideo1DTO, data: GenerateVideo1DTO,
speaking_controller: ISpeakingController = Depends(Provide[controller]) speaking_controller: ISpeakingController = Depends(Provide[controller])
): ):
return await speaking_controller.generate_video( return await speaking_controller.generate_video(
1, data.avatar, data.first_topic, data.questions, second_topic=data.second_topic 1, data.avatar, data.first_topic, data.questions, second_topic=data.second_topic
) )
@speaking_router.post( @speaking_router.post(
'/generate_video/2', '/generate_video/2',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def generate_video_2( async def generate_video_2(
data: GenerateVideo2DTO, data: GenerateVideo2DTO,
speaking_controller: ISpeakingController = Depends(Provide[controller]) speaking_controller: ISpeakingController = Depends(Provide[controller])
): ):
return await speaking_controller.generate_video( return await speaking_controller.generate_video(
2, data.avatar, data.topic, [data.question], prompts=data.prompts, suffix=data.suffix 2, data.avatar, data.topic, [data.question], prompts=data.prompts, suffix=data.suffix
) )
@speaking_router.post( @speaking_router.post(
'/generate_video/3', '/generate_video/3',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def generate_video_3( async def generate_video_3(
data: GenerateVideo3DTO, data: GenerateVideo3DTO,
speaking_controller: ISpeakingController = Depends(Provide[controller]) speaking_controller: ISpeakingController = Depends(Provide[controller])
): ):
return await speaking_controller.generate_video( return await speaking_controller.generate_video(
3, data.avatar, data.topic, data.questions 3, data.avatar, data.topic, data.questions
) )

68
app/api/training.py
View File

@@ -1,34 +1,34 @@
from dependency_injector.wiring import Provide, inject from dependency_injector.wiring import Provide, inject
from fastapi import APIRouter, Depends, Request from fastapi import APIRouter, Depends, Request
from app.dtos.training import FetchTipsDTO from app.dtos.training import FetchTipsDTO
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
from app.controllers.abc import ITrainingController from app.controllers.abc import ITrainingController
controller = "training_controller" controller = "training_controller"
training_router = APIRouter() training_router = APIRouter()
@training_router.post( @training_router.post(
'/tips', '/tips',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def get_reading_passage( async def get_reading_passage(
data: FetchTipsDTO, data: FetchTipsDTO,
training_controller: ITrainingController = Depends(Provide[controller]) training_controller: ITrainingController = Depends(Provide[controller])
): ):
return await training_controller.fetch_tips(data) return await training_controller.fetch_tips(data)
@training_router.post( @training_router.post(
'/', '/',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def training_content( async def training_content(
request: Request, request: Request,
training_controller: ITrainingController = Depends(Provide[controller]) training_controller: ITrainingController = Depends(Provide[controller])
): ):
data = await request.json() data = await request.json()
return await training_controller.get_training_content(data) return await training_controller.get_training_content(data)

21
app/api/user.py Normal file
View File

@@ -0,0 +1,21 @@
from dependency_injector.wiring import Provide, inject
from fastapi import APIRouter, Depends
from app.dtos.user_batch import BatchUsersDTO
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
from app.controllers.abc import IUserController
controller = "user_controller"
user_router = APIRouter()
@user_router.post(
'/import',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
)
@inject
async def batch_import(
batch: BatchUsersDTO,
user_controller: IUserController = Depends(Provide[controller])
):
return await user_controller.batch_import(batch)

50
app/api/writing.py
View File

@@ -1,25 +1,25 @@
import random import random
from dependency_injector.wiring import inject, Provide from dependency_injector.wiring import inject, Provide
from fastapi import APIRouter, Path, Query, Depends from fastapi import APIRouter, Path, Query, Depends
from app.middlewares import Authorized, IsAuthenticatedViaBearerToken from app.middlewares import Authorized, IsAuthenticatedViaBearerToken
from app.configs.constants import EducationalContent from app.configs.constants import EducationalContent
from app.controllers.abc import IWritingController from app.controllers.abc import IWritingController
controller = "writing_controller" controller = "writing_controller"
writing_router = APIRouter() writing_router = APIRouter()
@writing_router.get( @writing_router.get(
'/{task}', '/{task}',
dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))] dependencies=[Depends(Authorized([IsAuthenticatedViaBearerToken]))]
) )
@inject @inject
async def get_writing_task_general_question( async def get_writing_task_general_question(
task: int = Path(..., ge=1, le=2), task: int = Path(..., ge=1, le=2),
topic: str = Query(default=random.choice(EducationalContent.MTI_TOPICS)), topic: str = Query(default=random.choice(EducationalContent.MTI_TOPICS)),
difficulty: str = Query(default=random.choice(EducationalContent.DIFFICULTIES)), difficulty: str = Query(default=random.choice(EducationalContent.DIFFICULTIES)),
writing_controller: IWritingController = Depends(Provide[controller]) writing_controller: IWritingController = Depends(Provide[controller])
): ):
return await writing_controller.get_writing_task_general_question(task, topic, difficulty) return await writing_controller.get_writing_task_general_question(task, topic, difficulty)

10
app/configs/__init__.py
View File

@@ -1,5 +1,5 @@
from .dependency_injection import config_di from .dependency_injection import DependencyInjector
__all__ = [ __all__ = [
"config_di" "DependencyInjector"
] ]

1494
app/configs/constants.py
View File

File diff suppressed because it is too large Load Diff

260
app/configs/dependency_injection.py
View File

@@ -1,120 +1,140 @@
import json import json
import os import os
from dependency_injector import providers, containers from dependency_injector import providers, containers
from firebase_admin import credentials from firebase_admin import credentials
from openai import AsyncOpenAI from motor.motor_asyncio import AsyncIOMotorClient
from httpx import AsyncClient as HTTPClient from openai import AsyncOpenAI
from google.cloud.firestore_v1 import AsyncClient as FirestoreClient from httpx import AsyncClient as HTTPClient
from dotenv import load_dotenv from dotenv import load_dotenv
from sentence_transformers import SentenceTransformer from sentence_transformers import SentenceTransformer
from app.repositories.impl import * from app.repositories.impl import *
from app.services.impl import * from app.services.impl import *
from app.controllers.impl import * from app.controllers.impl import *
load_dotenv() load_dotenv()
def config_di( class DependencyInjector:
*, polly_client: any, http_client: HTTPClient, whisper_model: any
) -> None: def __init__(self, polly_client: any, http_client: HTTPClient, whisper_model: any):
""" self._container = containers.DynamicContainer()
Loads up all the common configs of all the environments self._polly_client = polly_client
and then calls the specific env configs self._http_client = http_client
""" self._whisper_model = whisper_model
# Firebase token
cred = credentials.Certificate(os.getenv("GOOGLE_APPLICATION_CREDENTIALS")) def inject(self):
firebase_token = cred.get_access_token().access_token self._setup_clients()
self._setup_third_parties()
container = containers.DynamicContainer() self._setup_repositories()
self._setup_services()
openai_client = providers.Singleton(AsyncOpenAI) self._setup_controllers()
polly_client = providers.Object(polly_client) self._container.wire(
http_client = providers.Object(http_client) packages=["app"]
firestore_client = providers.Singleton(FirestoreClient) )
whisper_model = providers.Object(whisper_model)
def _setup_clients(self):
llm = providers.Factory(OpenAI, client=openai_client) self._container.openai_client = providers.Singleton(AsyncOpenAI)
stt = providers.Factory(OpenAIWhisper, model=whisper_model) self._container.polly_client = providers.Object(self._polly_client)
tts = providers.Factory(AWSPolly, client=polly_client) self._container.http_client = providers.Object(self._http_client)
vid_gen = providers.Factory(Heygen, client=http_client, heygen_token=os.getenv("HEY_GEN_TOKEN")) self._container.whisper_model = providers.Object(self._whisper_model)
ai_detector = providers.Factory(GPTZero, client=http_client, gpt_zero_key=os.getenv("GPT_ZERO_API_KEY"))
def _setup_third_parties(self):
firebase_instance = providers.Factory( self._container.llm = providers.Factory(OpenAI, client=self._container.openai_client)
FirebaseStorage, client=http_client, token=firebase_token, bucket=os.getenv("FIREBASE_BUCKET") self._container.stt = providers.Factory(OpenAIWhisper, model=self._container.whisper_model)
) self._container.tts = providers.Factory(AWSPolly, client=self._container.polly_client)
self._container.vid_gen = providers.Factory(
firestore = providers.Factory(Firestore, client=firestore_client) Heygen, client=self._container.http_client, heygen_token=os.getenv("HEY_GEN_TOKEN")
)
# Services self._container.ai_detector = providers.Factory(
GPTZero, client=self._container.http_client, gpt_zero_key=os.getenv("GPT_ZERO_API_KEY")
listening_service = providers.Factory( )
ListeningService, llm=llm, tts=tts, file_storage=firebase_instance, document_store=firestore
) def _setup_repositories(self):
reading_service = providers.Factory(ReadingService, llm=llm) cred = credentials.Certificate(os.getenv("GOOGLE_APPLICATION_CREDENTIALS"))
firebase_token = cred.get_access_token().access_token
speaking_service = providers.Factory(
SpeakingService, llm=llm, vid_gen=vid_gen, self._container.document_store = providers.Object(
file_storage=firebase_instance, document_store=firestore, AsyncIOMotorClient(os.getenv("MONGODB_URI"))[os.getenv("MONGODB_DB")]
stt=stt )
)
self._container.firebase_instance = providers.Factory(
writing_service = providers.Factory(WritingService, llm=llm, ai_detector=ai_detector) FirebaseStorage,
client=self._container.http_client, token=firebase_token, bucket=os.getenv("FIREBASE_BUCKET")
with open('app/services/impl/level/mc_variants.json', 'r') as file: )
mc_variants = json.load(file)
def _setup_services(self):
level_service = providers.Factory( self._container.listening_service = providers.Factory(
LevelService, llm=llm, document_store=firestore, mc_variants=mc_variants, reading_service=reading_service, ListeningService,
writing_service=writing_service, speaking_service=speaking_service, listening_service=listening_service llm=self._container.llm,
) tts=self._container.tts,
file_storage=self._container.firebase_instance,
grade_service = providers.Factory( document_store=self._container.document_store
GradeService, llm=llm )
) self._container.reading_service = providers.Factory(ReadingService, llm=self._container.llm)
embeddings = SentenceTransformer('all-MiniLM-L6-v2') self._container.speaking_service = providers.Factory(
SpeakingService, llm=self._container.llm, vid_gen=self._container.vid_gen,
training_kb = providers.Factory( file_storage=self._container.firebase_instance, document_store=self._container.document_store,
TrainingContentKnowledgeBase, embeddings=embeddings stt=self._container.stt
) )
training_service = providers.Factory( self._container.writing_service = providers.Factory(
TrainingService, llm=llm, firestore=firestore, training_kb=training_kb WritingService, llm=self._container.llm, ai_detector=self._container.ai_detector
) )
# Controllers with open('app/services/impl/exam/level/mc_variants.json', 'r') as file:
mc_variants = json.load(file)
container.grade_controller = providers.Factory(
GradeController, grade_service=grade_service, speaking_service=speaking_service, writing_service=writing_service self._container.level_service = providers.Factory(
) LevelService, llm=self._container.llm, document_store=self._container.document_store,
mc_variants=mc_variants, reading_service=self._container.reading_service,
container.training_controller = providers.Factory( writing_service=self._container.writing_service, speaking_service=self._container.speaking_service,
TrainingController, training_service=training_service listening_service=self._container.listening_service
) )
container.level_controller = providers.Factory( self._container.grade_service = providers.Factory(
LevelController, level_service=level_service GradeService, llm=self._container.llm
) )
container.listening_controller = providers.Factory(
ListeningController, listening_service=listening_service embeddings = SentenceTransformer('all-MiniLM-L6-v2')
)
self._container.training_kb = providers.Factory(
container.reading_controller = providers.Factory( TrainingContentKnowledgeBase, embeddings=embeddings
ReadingController, reading_service=reading_service )
)
self._container.training_service = providers.Factory(
container.speaking_controller = providers.Factory( TrainingService, llm=self._container.llm,
SpeakingController, speaking_service=speaking_service firestore=self._container.document_store, training_kb=self._container.training_kb
) )
container.writing_controller = providers.Factory( def _setup_controllers(self):
WritingController, writing_service=writing_service self._container.grade_controller = providers.Factory(
) GradeController, grade_service=self._container.grade_service,
speaking_service=self._container.speaking_service,
container.llm = llm writing_service=self._container.writing_service
)
container.wire(
packages=["app"] self._container.training_controller = providers.Factory(
) TrainingController, training_service=self._container.training_service
)
self._container.level_controller = providers.Factory(
LevelController, level_service=self._container.level_service
)
self._container.listening_controller = providers.Factory(
ListeningController, listening_service=self._container.listening_service
)
self._container.reading_controller = providers.Factory(
ReadingController, reading_service=self._container.reading_service
)
self._container.speaking_controller = providers.Factory(
SpeakingController, speaking_service=self._container.speaking_service
)
self._container.writing_controller = providers.Factory(
WritingController, writing_service=self._container.writing_service
)

14
app/configs/logging/__init__.py
View File

@@ -1,7 +1,7 @@
from .filters import ErrorAndAboveFilter from .filters import ErrorAndAboveFilter
from .queue_handler import QueueListenerHandler from .queue_handler import QueueListenerHandler
__all__ = [ __all__ = [
"ErrorAndAboveFilter", "ErrorAndAboveFilter",
"QueueListenerHandler" "QueueListenerHandler"
] ]

12
app/configs/logging/filters.py
View File

@@ -1,6 +1,6 @@
import logging import logging
class ErrorAndAboveFilter(logging.Filter): class ErrorAndAboveFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord: def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord:
return record.levelno < logging.ERROR return record.levelno < logging.ERROR

210
app/configs/logging/formatters.py
View File

@@ -1,105 +1,105 @@
import datetime as dt import datetime as dt
import json import json
import logging import logging
LOG_RECORD_BUILTIN_ATTRS = { LOG_RECORD_BUILTIN_ATTRS = {
"args", "args",
"asctime", "asctime",
"created", "created",
"exc_info", "exc_info",
"exc_text", "exc_text",
"filename", "filename",
"funcName", "funcName",
"levelname", "levelname",
"levelno", "levelno",
"lineno", "lineno",
"module", "module",
"msecs", "msecs",
"message", "message",
"msg", "msg",
"name", "name",
"pathname", "pathname",
"process", "process",
"processName", "processName",
"relativeCreated", "relativeCreated",
"stack_info", "stack_info",
"thread", "thread",
"threadName", "threadName",
"taskName", "taskName",
} }
""" """
This isn't being used since the app will be run on gcloud run but this can be used for future apps. This isn't being used since the app will be run on gcloud run but this can be used for future apps.
If you want to test it: If you want to test it:
formatters: formatters:
"json": { "json": {
"()": "json_formatter.JSONFormatter", "()": "json_formatter.JSONFormatter",
"fmt_keys": { "fmt_keys": {
"level": "levelname", "level": "levelname",
"message": "message", "message": "message",
"timestamp": "timestamp", "timestamp": "timestamp",
"logger": "name", "logger": "name",
"module": "module", "module": "module",
"function": "funcName", "function": "funcName",
"line": "lineno", "line": "lineno",
"thread_name": "threadName" "thread_name": "threadName"
} }
} }
handlers: handlers:
"file_json": { "file_json": {
"class": "logging.handlers.RotatingFileHandler", "class": "logging.handlers.RotatingFileHandler",
"level": "DEBUG", "level": "DEBUG",
"formatter": "json", "formatter": "json",
"filename": "logs/log", "filename": "logs/log",
"maxBytes": 1000000, "maxBytes": 1000000,
"backupCount": 3 "backupCount": 3
} }
and add "cfg://handlers.file_json" to queue handler and add "cfg://handlers.file_json" to queue handler
""" """
# From this video https://www.youtube.com/watch?v=9L77QExPmI0 # From this video https://www.youtube.com/watch?v=9L77QExPmI0
# Src here: https://github.com/mCodingLLC/VideosSampleCode/blob/master/videos/135_modern_logging/mylogger.py # Src here: https://github.com/mCodingLLC/VideosSampleCode/blob/master/videos/135_modern_logging/mylogger.py
class JSONFormatter(logging.Formatter): class JSONFormatter(logging.Formatter):
def __init__( def __init__(
self, self,
*, *,
fmt_keys: dict[str, str] | None = None, fmt_keys: dict[str, str] | None = None,
): ):
super().__init__() super().__init__()
self.fmt_keys = fmt_keys if fmt_keys is not None else {} self.fmt_keys = fmt_keys if fmt_keys is not None else {}
def format(self, record: logging.LogRecord) -> str: def format(self, record: logging.LogRecord) -> str:
message = self._prepare_log_dict(record) message = self._prepare_log_dict(record)
return json.dumps(message, default=str) return json.dumps(message, default=str)
def _prepare_log_dict(self, record: logging.LogRecord): def _prepare_log_dict(self, record: logging.LogRecord):
always_fields = { always_fields = {
"message": record.getMessage(), "message": record.getMessage(),
"timestamp": dt.datetime.fromtimestamp( "timestamp": dt.datetime.fromtimestamp(
record.created, tz=dt.timezone.utc record.created, tz=dt.timezone.utc
).isoformat(), ).isoformat(),
} }
if record.exc_info is not None: if record.exc_info is not None:
always_fields["exc_info"] = self.formatException(record.exc_info) always_fields["exc_info"] = self.formatException(record.exc_info)
if record.stack_info is not None: if record.stack_info is not None:
always_fields["stack_info"] = self.formatStack(record.stack_info) always_fields["stack_info"] = self.formatStack(record.stack_info)
message = { message = {
key: msg_val key: msg_val
if (msg_val := always_fields.pop(val, None)) is not None if (msg_val := always_fields.pop(val, None)) is not None
else getattr(record, val) else getattr(record, val)
for key, val in self.fmt_keys.items() for key, val in self.fmt_keys.items()
} }
message.update(always_fields) message.update(always_fields)
for key, val in record.__dict__.items(): for key, val in record.__dict__.items():
if key not in LOG_RECORD_BUILTIN_ATTRS: if key not in LOG_RECORD_BUILTIN_ATTRS:
message[key] = val message[key] = val
return message return message

106
app/configs/logging/logging_config.json
View File

@@ -1,53 +1,53 @@
{ {
"version": 1, "version": 1,
"objects": { "objects": {
"queue": { "queue": {
"class": "queue.Queue", "class": "queue.Queue",
"maxsize": 1000 "maxsize": 1000
} }
}, },
"disable_existing_loggers": false, "disable_existing_loggers": false,
"formatters": { "formatters": {
"simple": { "simple": {
"format": "[%(levelname)s] (%(module)s|L: %(lineno)d) %(asctime)s: %(message)s", "format": "[%(levelname)s] (%(module)s|L: %(lineno)d) %(asctime)s: %(message)s",
"datefmt": "%Y-%m-%dT%H:%M:%S%z" "datefmt": "%Y-%m-%dT%H:%M:%S%z"
} }
}, },
"filters": { "filters": {
"error_and_above": { "error_and_above": {
"()": "app.configs.logging.ErrorAndAboveFilter" "()": "app.configs.logging.ErrorAndAboveFilter"
} }
}, },
"handlers": { "handlers": {
"console": { "console": {
"class": "logging.StreamHandler", "class": "logging.StreamHandler",
"level": "INFO", "level": "INFO",
"formatter": "simple", "formatter": "simple",
"stream": "ext://sys.stdout", "stream": "ext://sys.stdout",
"filters": ["error_and_above"] "filters": ["error_and_above"]
}, },
"error": { "error": {
"class": "logging.StreamHandler", "class": "logging.StreamHandler",
"level": "ERROR", "level": "ERROR",
"formatter": "simple", "formatter": "simple",
"stream": "ext://sys.stderr" "stream": "ext://sys.stderr"
}, },
"queue_handler": { "queue_handler": {
"class": "app.configs.logging.QueueListenerHandler", "class": "app.configs.logging.QueueListenerHandler",
"handlers": [ "handlers": [
"cfg://handlers.console", "cfg://handlers.console",
"cfg://handlers.error" "cfg://handlers.error"
], ],
"queue": "cfg://objects.queue", "queue": "cfg://objects.queue",
"respect_handler_level": true "respect_handler_level": true
} }
}, },
"loggers": { "loggers": {
"root": { "root": {
"level": "DEBUG", "level": "DEBUG",
"handlers": [ "handlers": [
"queue_handler" "queue_handler"
] ]
} }
} }
} }

122
app/configs/logging/queue_handler.py
View File

@@ -1,61 +1,61 @@
from logging.config import ConvertingList, ConvertingDict, valid_ident from logging.config import ConvertingList, ConvertingDict, valid_ident
from logging.handlers import QueueHandler, QueueListener from logging.handlers import QueueHandler, QueueListener
from queue import Queue from queue import Queue
import atexit import atexit
class QueueHnadlerHelper: class QueueHnadlerHelper:
@staticmethod @staticmethod
def resolve_handlers(l): def resolve_handlers(l):
if not isinstance(l, ConvertingList): if not isinstance(l, ConvertingList):
return l return l
# Indexing the list performs the evaluation. # Indexing the list performs the evaluation.
return [l[i] for i in range(len(l))] return [l[i] for i in range(len(l))]
@staticmethod @staticmethod
def resolve_queue(q): def resolve_queue(q):
if not isinstance(q, ConvertingDict): if not isinstance(q, ConvertingDict):
return q return q
if '__resolved_value__' in q: if '__resolved_value__' in q:
return q['__resolved_value__'] return q['__resolved_value__']
cname = q.pop('class') cname = q.pop('class')
klass = q.configurator.resolve(cname) klass = q.configurator.resolve(cname)
props = q.pop('.', None) props = q.pop('.', None)
kwargs = {k: q[k] for k in q if valid_ident(k)} kwargs = {k: q[k] for k in q if valid_ident(k)}
result = klass(**kwargs) result = klass(**kwargs)
if props: if props:
for name, value in props.items(): for name, value in props.items():
setattr(result, name, value) setattr(result, name, value)
q['__resolved_value__'] = result q['__resolved_value__'] = result
return result return result
# The guy from this video https://www.youtube.com/watch?v=9L77QExPmI0 is using logging features only available in 3.12 # The guy from this video https://www.youtube.com/watch?v=9L77QExPmI0 is using logging features only available in 3.12
# This article had the class required to build the queue handler in 3.11 # This article had the class required to build the queue handler in 3.11
# https://rob-blackbourn.medium.com/how-to-use-python-logging-queuehandler-with-dictconfig-1e8b1284e27a # https://rob-blackbourn.medium.com/how-to-use-python-logging-queuehandler-with-dictconfig-1e8b1284e27a
class QueueListenerHandler(QueueHandler): class QueueListenerHandler(QueueHandler):
def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)): def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)):
queue = QueueHnadlerHelper.resolve_queue(queue) queue = QueueHnadlerHelper.resolve_queue(queue)
super().__init__(queue) super().__init__(queue)
handlers = QueueHnadlerHelper.resolve_handlers(handlers) handlers = QueueHnadlerHelper.resolve_handlers(handlers)
self._listener = QueueListener( self._listener = QueueListener(
self.queue, self.queue,
*handlers, *handlers,
respect_handler_level=respect_handler_level) respect_handler_level=respect_handler_level)
if auto_run: if auto_run:
self.start() self.start()
atexit.register(self.stop) atexit.register(self.stop)
def start(self): def start(self):
self._listener.start() self._listener.start()
def stop(self): def stop(self):
self._listener.stop() self._listener.stop()
def emit(self, record): def emit(self, record):
return super().emit(record) return super().emit(record)

2550
app/configs/question_templates.py
View File

File diff suppressed because it is too large Load Diff

36
app/controllers/abc/__init__.py
View File

@@ -1,17 +1,19 @@
from .level import ILevelController from .level import ILevelController
from .listening import IListeningController from .listening import IListeningController
from .reading import IReadingController from .reading import IReadingController
from .writing import IWritingController from .writing import IWritingController
from .speaking import ISpeakingController from .speaking import ISpeakingController
from .grade import IGradeController from .grade import IGradeController
from .training import ITrainingController from .training import ITrainingController
from .user import IUserController
__all__ = [
"IListeningController", __all__ = [
"IReadingController", "IListeningController",
"IWritingController", "IReadingController",
"ISpeakingController", "IWritingController",
"ILevelController", "ISpeakingController",
"IGradeController", "ILevelController",
"ITrainingController" "IGradeController",
] "ITrainingController",
"IUserController"
]

44
app/controllers/abc/grade.py
View File

@@ -1,22 +1,22 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Dict, List from typing import Dict, List
class IGradeController(ABC): class IGradeController(ABC):
@abstractmethod @abstractmethod
async def grade_writing_task(self, task: int, data): async def grade_writing_task(self, task: int, data):
pass pass
@abstractmethod @abstractmethod
async def grade_speaking_task(self, task: int, answers: List[Dict]) -> Dict: async def grade_speaking_task(self, task: int, answers: List[Dict]) -> Dict:
pass pass
@abstractmethod @abstractmethod
async def grade_short_answers(self, data: Dict): async def grade_short_answers(self, data: Dict):
pass pass
@abstractmethod @abstractmethod
async def grading_summary(self, data: Dict): async def grading_summary(self, data: Dict):
pass pass

46
app/controllers/abc/level.py
View File

@@ -1,23 +1,23 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from fastapi import UploadFile from fastapi import UploadFile
from typing import Dict from typing import Dict
class ILevelController(ABC): class ILevelController(ABC):
@abstractmethod @abstractmethod
async def get_level_exam(self): async def get_level_exam(self):
pass pass
@abstractmethod @abstractmethod
async def get_level_utas(self): async def get_level_utas(self):
pass pass
@abstractmethod @abstractmethod
async def upload_level(self, file: UploadFile): async def upload_level(self, file: UploadFile):
pass pass
@abstractmethod @abstractmethod
async def get_custom_level(self, data: Dict): async def get_custom_level(self, data: Dict):
pass pass

26
app/controllers/abc/listening.py
View File

@@ -1,13 +1,13 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List from typing import List
class IListeningController(ABC): class IListeningController(ABC):
@abstractmethod @abstractmethod
async def get_listening_question(self, section_id: int, topic: str, exercises: List[str], difficulty: str): async def get_listening_question(self, section_id: int, topic: str, exercises: List[str], difficulty: str):
pass pass
@abstractmethod @abstractmethod
async def save_listening(self, data): async def save_listening(self, data):
pass pass

20
app/controllers/abc/reading.py
View File

@@ -1,10 +1,10 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List from typing import List
class IReadingController(ABC): class IReadingController(ABC):
@abstractmethod @abstractmethod
async def get_reading_passage(self, passage: int, topic: str, exercises: List[str], difficulty: str): async def get_reading_passage(self, passage: int, topic: str, exercises: List[str], difficulty: str):
pass pass

50
app/controllers/abc/speaking.py
View File

@@ -1,25 +1,25 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Optional from typing import Optional
from fastapi import BackgroundTasks from fastapi import BackgroundTasks
class ISpeakingController(ABC): class ISpeakingController(ABC):
@abstractmethod @abstractmethod
async def get_speaking_part(self, task: int, topic: str, difficulty: str, second_topic: Optional[str] = None): async def get_speaking_part(self, task: int, topic: str, difficulty: str, second_topic: Optional[str] = None):
pass pass
@abstractmethod @abstractmethod
async def save_speaking(self, data, background_tasks: BackgroundTasks): async def save_speaking(self, data, background_tasks: BackgroundTasks):
pass pass
@abstractmethod @abstractmethod
async def generate_video( async def generate_video(
self, part: int, avatar: str, topic: str, questions: list[str], self, part: int, avatar: str, topic: str, questions: list[str],
*, *,
second_topic: Optional[str] = None, second_topic: Optional[str] = None,
prompts: Optional[list[str]] = None, prompts: Optional[list[str]] = None,
suffix: Optional[str] = None, suffix: Optional[str] = None,
): ):
pass pass

24
app/controllers/abc/training.py
View File

@@ -1,12 +1,12 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class ITrainingController(ABC): class ITrainingController(ABC):
@abstractmethod @abstractmethod
async def fetch_tips(self, data): async def fetch_tips(self, data):
pass pass
@abstractmethod @abstractmethod
async def get_training_content(self, data): async def get_training_content(self, data):
pass pass

10
app/controllers/abc/user.py Normal file
View File

@@ -0,0 +1,10 @@
from abc import ABC, abstractmethod
from app.dtos.user_batch import BatchUsersDTO
class IUserController(ABC):
@abstractmethod
async def batch_import(self, batch: BatchUsersDTO):
pass

16
app/controllers/abc/writing.py
View File

@@ -1,8 +1,8 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IWritingController(ABC): class IWritingController(ABC):
@abstractmethod @abstractmethod
async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str):
pass pass

36
app/controllers/impl/__init__.py
View File

@@ -1,17 +1,19 @@
from .level import LevelController from .level import LevelController
from .listening import ListeningController from .listening import ListeningController
from .reading import ReadingController from .reading import ReadingController
from .speaking import SpeakingController from .speaking import SpeakingController
from .writing import WritingController from .writing import WritingController
from .training import TrainingController from .training import TrainingController
from .grade import GradeController from .grade import GradeController
from .user import UserController
__all__ = [
"LevelController", __all__ = [
"ListeningController", "LevelController",
"ReadingController", "ListeningController",
"SpeakingController", "ReadingController",
"WritingController", "SpeakingController",
"TrainingController", "WritingController",
"GradeController" "TrainingController",
] "GradeController",
"UserController"
]

108
app/controllers/impl/grade.py
View File

@@ -1,54 +1,54 @@
import logging import logging
from typing import Dict, List from typing import Dict, List
from app.configs.constants import FilePaths from app.configs.constants import FilePaths
from app.controllers.abc import IGradeController from app.controllers.abc import IGradeController
from app.dtos.writing import WritingGradeTaskDTO from app.dtos.writing import WritingGradeTaskDTO
from app.helpers import FileHelper from app.helpers import FileHelper
from app.services.abc import ISpeakingService, IWritingService, IGradeService from app.services.abc import ISpeakingService, IWritingService, IGradeService
from app.utils import handle_exception from app.utils import handle_exception
class GradeController(IGradeController): class GradeController(IGradeController):
def __init__( def __init__(
self, self,
grade_service: IGradeService, grade_service: IGradeService,
speaking_service: ISpeakingService, speaking_service: ISpeakingService,
writing_service: IWritingService writing_service: IWritingService
): ):
self._service = grade_service self._service = grade_service
self._speaking_service = speaking_service self._speaking_service = speaking_service
self._writing_service = writing_service self._writing_service = writing_service
self._logger = logging.getLogger(__name__) self._logger = logging.getLogger(__name__)
async def grade_writing_task(self, task: int, data: WritingGradeTaskDTO): async def grade_writing_task(self, task: int, data: WritingGradeTaskDTO):
return await self._writing_service.grade_writing_task(task, data.question, data.answer) return await self._writing_service.grade_writing_task(task, data.question, data.answer)
@handle_exception(400) @handle_exception(400)
async def grade_speaking_task(self, task: int, answers: List[Dict]) -> Dict: async def grade_speaking_task(self, task: int, answers: List[Dict]) -> Dict:
FileHelper.delete_files_older_than_one_day(FilePaths.AUDIO_FILES_PATH) FileHelper.delete_files_older_than_one_day(FilePaths.AUDIO_FILES_PATH)
return await self._speaking_service.grade_speaking_task(task, answers) return await self._speaking_service.grade_speaking_task(task, answers)
async def grade_short_answers(self, data: Dict): async def grade_short_answers(self, data: Dict):
return await self._service.grade_short_answers(data) return await self._service.grade_short_answers(data)
async def grading_summary(self, data: Dict): async def grading_summary(self, data: Dict):
section_keys = ['reading', 'listening', 'writing', 'speaking', 'level'] section_keys = ['reading', 'listening', 'writing', 'speaking', 'level']
extracted_sections = self._extract_existing_sections_from_body(data, section_keys) extracted_sections = self._extract_existing_sections_from_body(data, section_keys)
return await self._service.calculate_grading_summary(extracted_sections) return await self._service.calculate_grading_summary(extracted_sections)
@staticmethod @staticmethod
def _extract_existing_sections_from_body(my_dict, keys_to_extract): def _extract_existing_sections_from_body(my_dict, keys_to_extract):
if 'sections' in my_dict and isinstance(my_dict['sections'], list) and len(my_dict['sections']) > 0: if 'sections' in my_dict and isinstance(my_dict['sections'], list) and len(my_dict['sections']) > 0:
return list( return list(
filter( filter(
lambda item: lambda item:
'code' in item and 'code' in item and
item['code'] in keys_to_extract and item['code'] in keys_to_extract and
'grade' in item and 'grade' in item and
'name' in item, 'name' in item,
my_dict['sections'] my_dict['sections']
) )
) )

46
app/controllers/impl/level.py
View File

@@ -1,23 +1,23 @@
from fastapi import UploadFile from fastapi import UploadFile
from typing import Dict from typing import Dict
from app.controllers.abc import ILevelController from app.controllers.abc import ILevelController
from app.services.abc import ILevelService from app.services.abc import ILevelService
class LevelController(ILevelController): class LevelController(ILevelController):
def __init__(self, level_service: ILevelService): def __init__(self, level_service: ILevelService):
self._service = level_service self._service = level_service
async def get_level_exam(self): async def get_level_exam(self):
return await self._service.get_level_exam() return await self._service.get_level_exam()
async def get_level_utas(self): async def get_level_utas(self):
return await self._service.get_level_utas() return await self._service.get_level_utas()
async def upload_level(self, file: UploadFile): async def upload_level(self, file: UploadFile):
return await self._service.upload_level(file) return await self._service.upload_level(file)
async def get_custom_level(self, data: Dict): async def get_custom_level(self, data: Dict):
return await self._service.get_custom_level(data) return await self._service.get_custom_level(data)

38
app/controllers/impl/listening.py
View File

@@ -1,19 +1,19 @@
from typing import List from typing import List
from app.controllers.abc import IListeningController from app.controllers.abc import IListeningController
from app.dtos.listening import SaveListeningDTO from app.dtos.listening import SaveListeningDTO
from app.services.abc import IListeningService from app.services.abc import IListeningService
class ListeningController(IListeningController): class ListeningController(IListeningController):
def __init__(self, listening_service: IListeningService): def __init__(self, listening_service: IListeningService):
self._service = listening_service self._service = listening_service
async def get_listening_question( async def get_listening_question(
self, section_id: int, topic: str, req_exercises: List[str], difficulty: str self, section_id: int, topic: str, req_exercises: List[str], difficulty: str
): ):
return await self._service.get_listening_question(section_id, topic, req_exercises, difficulty) return await self._service.get_listening_question(section_id, topic, req_exercises, difficulty)
async def save_listening(self, data: SaveListeningDTO): async def save_listening(self, data: SaveListeningDTO):
return await self._service.save_listening(data.parts, data.minTimer, data.difficulty, data.id) return await self._service.save_listening(data.parts, data.minTimer, data.difficulty, data.id)

86
app/controllers/impl/reading.py
View File

@@ -1,43 +1,43 @@
import random import random
import logging import logging
from typing import List from typing import List
from app.controllers.abc import IReadingController from app.controllers.abc import IReadingController
from app.services.abc import IReadingService from app.services.abc import IReadingService
from app.configs.constants import FieldsAndExercises from app.configs.constants import FieldsAndExercises
from app.helpers import ExercisesHelper from app.helpers import ExercisesHelper
class ReadingController(IReadingController): class ReadingController(IReadingController):
def __init__(self, reading_service: IReadingService): def __init__(self, reading_service: IReadingService):
self._service = reading_service self._service = reading_service
self._logger = logging.getLogger(__name__) self._logger = logging.getLogger(__name__)
self._passages = { self._passages = {
"passage_1": { "passage_1": {
"start_id": 1, "start_id": 1,
"total_exercises": FieldsAndExercises.TOTAL_READING_PASSAGE_1_EXERCISES "total_exercises": FieldsAndExercises.TOTAL_READING_PASSAGE_1_EXERCISES
}, },
"passage_2": { "passage_2": {
"start_id": 14, "start_id": 14,
"total_exercises": FieldsAndExercises.TOTAL_READING_PASSAGE_2_EXERCISES "total_exercises": FieldsAndExercises.TOTAL_READING_PASSAGE_2_EXERCISES
}, },
"passage_3": { "passage_3": {
"start_id": 27, "start_id": 27,
"total_exercises": FieldsAndExercises.TOTAL_READING_PASSAGE_3_EXERCISES "total_exercises": FieldsAndExercises.TOTAL_READING_PASSAGE_3_EXERCISES
} }
} }
async def get_reading_passage(self, passage_id: int, topic: str, req_exercises: List[str], difficulty: str): async def get_reading_passage(self, passage_id: int, topic: str, req_exercises: List[str], difficulty: str):
passage = self._passages[f'passage_{str(passage_id)}'] passage = self._passages[f'passage_{str(passage_id)}']
if len(req_exercises) == 0: if len(req_exercises) == 0:
req_exercises = random.sample(FieldsAndExercises.READING_EXERCISE_TYPES, 2) req_exercises = random.sample(FieldsAndExercises.READING_EXERCISE_TYPES, 2)
number_of_exercises_q = ExercisesHelper.divide_number_into_parts( number_of_exercises_q = ExercisesHelper.divide_number_into_parts(
passage["total_exercises"], len(req_exercises) passage["total_exercises"], len(req_exercises)
) )
return await self._service.gen_reading_passage( return await self._service.gen_reading_passage(
passage_id, topic, req_exercises, number_of_exercises_q, difficulty, passage["start_id"] passage_id, topic, req_exercises, number_of_exercises_q, difficulty, passage["start_id"]
) )

94
app/controllers/impl/speaking.py
View File

@@ -1,47 +1,47 @@
import logging import logging
import uuid import uuid
from typing import Optional from typing import Optional
from fastapi import BackgroundTasks from fastapi import BackgroundTasks
from app.controllers.abc import ISpeakingController from app.controllers.abc import ISpeakingController
from app.dtos.speaking import SaveSpeakingDTO from app.dtos.speaking import SaveSpeakingDTO
from app.services.abc import ISpeakingService from app.services.abc import ISpeakingService
from app.configs.constants import ExamVariant, MinTimers from app.configs.constants import ExamVariant, MinTimers
from app.configs.question_templates import getSpeakingTemplate from app.configs.question_templates import getSpeakingTemplate
class SpeakingController(ISpeakingController): class SpeakingController(ISpeakingController):
def __init__(self, speaking_service: ISpeakingService): def __init__(self, speaking_service: ISpeakingService):
self._service = speaking_service self._service = speaking_service
self._logger = logging.getLogger(__name__) self._logger = logging.getLogger(__name__)
async def get_speaking_part(self, task: int, topic: str, difficulty: str, second_topic: Optional[str] = None): async def get_speaking_part(self, task: int, topic: str, difficulty: str, second_topic: Optional[str] = None):
return await self._service.get_speaking_part(task, topic, difficulty, second_topic) return await self._service.get_speaking_part(task, topic, difficulty, second_topic)
async def save_speaking(self, data: SaveSpeakingDTO, background_tasks: BackgroundTasks): async def save_speaking(self, data: SaveSpeakingDTO, background_tasks: BackgroundTasks):
exercises = data.exercises exercises = data.exercises
min_timer = data.minTimer min_timer = data.minTimer
template = getSpeakingTemplate() template = getSpeakingTemplate()
template["minTimer"] = min_timer template["minTimer"] = min_timer
if min_timer < MinTimers.SPEAKING_MIN_TIMER_DEFAULT: if min_timer < MinTimers.SPEAKING_MIN_TIMER_DEFAULT:
template["variant"] = ExamVariant.PARTIAL.value template["variant"] = ExamVariant.PARTIAL.value
else: else:
template["variant"] = ExamVariant.FULL.value template["variant"] = ExamVariant.FULL.value
req_id = str(uuid.uuid4()) req_id = str(uuid.uuid4())
self._logger.info(f'Received request to save speaking with id: {req_id}') self._logger.info(f'Received request to save speaking with id: {req_id}')
background_tasks.add_task(self._service.create_videos_and_save_to_db, exercises, template, req_id) background_tasks.add_task(self._service.create_videos_and_save_to_db, exercises, template, req_id)
self._logger.info('Started background task to save speaking.') self._logger.info('Started background task to save speaking.')
# Return response without waiting for create_videos_and_save_to_db to finish # Return response without waiting for create_videos_and_save_to_db to finish
return {**template, "id": req_id} return {**template, "id": req_id}
async def generate_video(self, *args, **kwargs): async def generate_video(self, *args, **kwargs):
return await self._service.generate_video(*args, **kwargs) return await self._service.generate_video(*args, **kwargs)

34
app/controllers/impl/training.py
View File

@@ -1,17 +1,17 @@
from typing import Dict from typing import Dict
from app.controllers.abc import ITrainingController from app.controllers.abc import ITrainingController
from app.dtos.training import FetchTipsDTO from app.dtos.training import FetchTipsDTO
from app.services.abc import ITrainingService from app.services.abc import ITrainingService
class TrainingController(ITrainingController): class TrainingController(ITrainingController):
def __init__(self, training_service: ITrainingService): def __init__(self, training_service: ITrainingService):
self._service = training_service self._service = training_service
async def fetch_tips(self, data: FetchTipsDTO): async def fetch_tips(self, data: FetchTipsDTO):
return await self._service.fetch_tips(data.context, data.question, data.answer, data.correct_answer) return await self._service.fetch_tips(data.context, data.question, data.answer, data.correct_answer)
async def get_training_content(self, data: Dict): async def get_training_content(self, data: Dict):
return await self._service.get_training_content(data) return await self._service.get_training_content(data)

12
app/controllers/impl/user.py Normal file
View File

@@ -0,0 +1,12 @@
from app.controllers.abc import IUserController
from app.dtos.user_batch import BatchUsersDTO
from app.services.abc import IUserService
class UserController(IUserController):
def __init__(self, user_service: IUserService):
self._service = user_service
async def batch_import(self, batch: BatchUsersDTO):
return await self._service.fetch_tips(batch)

22
app/controllers/impl/writing.py
View File

@@ -1,11 +1,11 @@
from app.controllers.abc import IWritingController from app.controllers.abc import IWritingController
from app.services.abc import IWritingService from app.services.abc import IWritingService
class WritingController(IWritingController): class WritingController(IWritingController):
def __init__(self, writing_service: IWritingService): def __init__(self, writing_service: IWritingService):
self._service = writing_service self._service = writing_service
async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str):
return await self._service.get_writing_task_general_question(task, topic, difficulty) return await self._service.get_writing_task_general_question(task, topic, difficulty)

114
app/dtos/exam.py
View File

@@ -1,57 +1,57 @@
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from typing import List, Dict, Union, Optional from typing import List, Dict, Union, Optional
from uuid import uuid4, UUID from uuid import uuid4, UUID
class Option(BaseModel): class Option(BaseModel):
id: str id: str
text: str text: str
class MultipleChoiceQuestion(BaseModel): class MultipleChoiceQuestion(BaseModel):
id: str id: str
prompt: str prompt: str
variant: str = "text" variant: str = "text"
solution: str solution: str
options: List[Option] options: List[Option]
class MultipleChoiceExercise(BaseModel): class MultipleChoiceExercise(BaseModel):
id: UUID = Field(default_factory=uuid4) id: UUID = Field(default_factory=uuid4)
type: str = "multipleChoice" type: str = "multipleChoice"
prompt: str = "Select the appropriate option." prompt: str = "Select the appropriate option."
questions: List[MultipleChoiceQuestion] questions: List[MultipleChoiceQuestion]
userSolutions: List = Field(default_factory=list) userSolutions: List = Field(default_factory=list)
class FillBlanksWord(BaseModel): class FillBlanksWord(BaseModel):
id: str id: str
options: Dict[str, str] options: Dict[str, str]
class FillBlanksSolution(BaseModel): class FillBlanksSolution(BaseModel):
id: str id: str
solution: str solution: str
class FillBlanksExercise(BaseModel): class FillBlanksExercise(BaseModel):
id: UUID = Field(default_factory=uuid4) id: UUID = Field(default_factory=uuid4)
type: str = "fillBlanks" type: str = "fillBlanks"
variant: str = "mc" variant: str = "mc"
prompt: str = "Click a blank to select the appropriate word for it." prompt: str = "Click a blank to select the appropriate word for it."
text: str text: str
solutions: List[FillBlanksSolution] solutions: List[FillBlanksSolution]
words: List[FillBlanksWord] words: List[FillBlanksWord]
userSolutions: List = Field(default_factory=list) userSolutions: List = Field(default_factory=list)
Exercise = Union[MultipleChoiceExercise, FillBlanksExercise] Exercise = Union[MultipleChoiceExercise, FillBlanksExercise]
class Part(BaseModel): class Part(BaseModel):
exercises: List[Exercise] exercises: List[Exercise]
context: Optional[str] = Field(default=None) context: Optional[str] = Field(default=None)
class Exam(BaseModel): class Exam(BaseModel):
parts: List[Part] parts: List[Part]

28
app/dtos/listening.py
View File

@@ -1,14 +1,14 @@
import random import random
import uuid import uuid
from typing import List, Dict from typing import List, Dict
from pydantic import BaseModel from pydantic import BaseModel
from app.configs.constants import MinTimers, EducationalContent from app.configs.constants import MinTimers, EducationalContent
class SaveListeningDTO(BaseModel): class SaveListeningDTO(BaseModel):
parts: List[Dict] parts: List[Dict]
minTimer: int = MinTimers.LISTENING_MIN_TIMER_DEFAULT minTimer: int = MinTimers.LISTENING_MIN_TIMER_DEFAULT
difficulty: str = random.choice(EducationalContent.DIFFICULTIES) difficulty: str = random.choice(EducationalContent.DIFFICULTIES)
id: str = str(uuid.uuid4()) id: str = str(uuid.uuid4())

58
app/dtos/sheet.py
View File

@@ -1,29 +1,29 @@
from pydantic import BaseModel from pydantic import BaseModel
from typing import List, Dict, Union, Any, Optional from typing import List, Dict, Union, Any, Optional
class Option(BaseModel): class Option(BaseModel):
id: str id: str
text: str text: str
class MultipleChoiceQuestion(BaseModel): class MultipleChoiceQuestion(BaseModel):
type: str = "multipleChoice" type: str = "multipleChoice"
id: str id: str
prompt: str prompt: str
variant: str = "text" variant: str = "text"
options: List[Option] options: List[Option]
class FillBlanksWord(BaseModel): class FillBlanksWord(BaseModel):
type: str = "fillBlanks" type: str = "fillBlanks"
id: str id: str
options: Dict[str, str] options: Dict[str, str]
Component = Union[MultipleChoiceQuestion, FillBlanksWord, Dict[str, Any]] Component = Union[MultipleChoiceQuestion, FillBlanksWord, Dict[str, Any]]
class Sheet(BaseModel): class Sheet(BaseModel):
batch: Optional[int] = None batch: Optional[int] = None
components: List[Component] components: List[Component]

84
app/dtos/speaking.py
View File

@@ -1,42 +1,42 @@
import random import random
from typing import List, Dict from typing import List, Dict
from pydantic import BaseModel from pydantic import BaseModel
from app.configs.constants import MinTimers, AvatarEnum from app.configs.constants import MinTimers, AvatarEnum
class SaveSpeakingDTO(BaseModel): class SaveSpeakingDTO(BaseModel):
exercises: List[Dict] exercises: List[Dict]
minTimer: int = MinTimers.SPEAKING_MIN_TIMER_DEFAULT minTimer: int = MinTimers.SPEAKING_MIN_TIMER_DEFAULT
class GradeSpeakingDTO(BaseModel): class GradeSpeakingDTO(BaseModel):
question: str question: str
answer: str answer: str
class GradeSpeakingAnswersDTO(BaseModel): class GradeSpeakingAnswersDTO(BaseModel):
answers: List[Dict] answers: List[Dict]
class GenerateVideo1DTO(BaseModel): class GenerateVideo1DTO(BaseModel):
avatar: str = (random.choice(list(AvatarEnum))).value avatar: str = (random.choice(list(AvatarEnum))).value
questions: List[str] questions: List[str]
first_topic: str first_topic: str
second_topic: str second_topic: str
class GenerateVideo2DTO(BaseModel): class GenerateVideo2DTO(BaseModel):
avatar: str = (random.choice(list(AvatarEnum))).value avatar: str = (random.choice(list(AvatarEnum))).value
prompts: List[str] = [] prompts: List[str] = []
suffix: str = "" suffix: str = ""
question: str question: str
topic: str topic: str
class GenerateVideo3DTO(BaseModel): class GenerateVideo3DTO(BaseModel):
avatar: str = (random.choice(list(AvatarEnum))).value avatar: str = (random.choice(list(AvatarEnum))).value
questions: List[str] questions: List[str]
topic: str topic: str

74
app/dtos/training.py
View File

@@ -1,37 +1,37 @@
from pydantic import BaseModel from pydantic import BaseModel
from typing import List from typing import List
class FetchTipsDTO(BaseModel): class FetchTipsDTO(BaseModel):
context: str context: str
question: str question: str
answer: str answer: str
correct_answer: str correct_answer: str
class QueryDTO(BaseModel): class QueryDTO(BaseModel):
category: str category: str
text: str text: str
class DetailsDTO(BaseModel): class DetailsDTO(BaseModel):
exam_id: str exam_id: str
date: int date: int
performance_comment: str performance_comment: str
detailed_summary: str detailed_summary: str
class WeakAreaDTO(BaseModel): class WeakAreaDTO(BaseModel):
area: str area: str
comment: str comment: str
class TrainingContentDTO(BaseModel): class TrainingContentDTO(BaseModel):
details: List[DetailsDTO] details: List[DetailsDTO]
weak_areas: List[WeakAreaDTO] weak_areas: List[WeakAreaDTO]
queries: List[QueryDTO] queries: List[QueryDTO]
class TipsDTO(BaseModel): class TipsDTO(BaseModel):
tip_ids: List[str] tip_ids: List[str]

30
app/dtos/user_batch.py Normal file
View File

@@ -0,0 +1,30 @@
import uuid
from typing import Optional
from pydantic import BaseModel, Field
class DemographicInfo(BaseModel):
phone: str
passport_id: Optional[str] = None
country: Optional[str] = None
class UserDTO(BaseModel):
id: uuid.UUID = Field(default_factory=uuid.uuid4)
email: str
name: str
type: str
passport_id: str
passwordHash: str
passwordSalt: str
groupName: Optional[str] = None
corporate: Optional[str] = None
studentID: Optional[str | int] = None
expiryDate: Optional[str] = None
demographicInformation: Optional[DemographicInfo] = None
class BatchUsersDTO(BaseModel):
makerID: str
users: list[UserDTO]

12
app/dtos/writing.py
View File

@@ -1,6 +1,6 @@
from pydantic import BaseModel from pydantic import BaseModel
class WritingGradeTaskDTO(BaseModel): class WritingGradeTaskDTO(BaseModel):
question: str question: str
answer: str answer: str

12
app/exceptions/__init__.py
View File

@@ -1,6 +1,6 @@
from .exceptions import CustomException, UnauthorizedException from .exceptions import CustomException, UnauthorizedException
__all__ = [ __all__ = [
"CustomException", "CustomException",
"UnauthorizedException" "UnauthorizedException"
] ]

34
app/exceptions/exceptions.py
View File

@@ -1,17 +1,17 @@
from http import HTTPStatus from http import HTTPStatus
class CustomException(Exception): class CustomException(Exception):
code = HTTPStatus.INTERNAL_SERVER_ERROR code = HTTPStatus.INTERNAL_SERVER_ERROR
error_code = HTTPStatus.INTERNAL_SERVER_ERROR error_code = HTTPStatus.INTERNAL_SERVER_ERROR
message = HTTPStatus.INTERNAL_SERVER_ERROR.description message = HTTPStatus.INTERNAL_SERVER_ERROR.description
def __init__(self, message=None): def __init__(self, message=None):
if message: if message:
self.message = message self.message = message
class UnauthorizedException(CustomException): class UnauthorizedException(CustomException):
code = HTTPStatus.UNAUTHORIZED code = HTTPStatus.UNAUTHORIZED
error_code = HTTPStatus.UNAUTHORIZED error_code = HTTPStatus.UNAUTHORIZED
message = HTTPStatus.UNAUTHORIZED.description message = HTTPStatus.UNAUTHORIZED.description

26
app/helpers/__init__.py
View File

@@ -1,13 +1,13 @@
from .file import FileHelper from .file import FileHelper
from .text import TextHelper from .text import TextHelper
from .token_counter import count_tokens from .token_counter import count_tokens
from .exercises import ExercisesHelper from .exercises import ExercisesHelper
from .logger import LoggerHelper from .logger import LoggerHelper
__all__ = [ __all__ = [
"FileHelper", "FileHelper",
"TextHelper", "TextHelper",
"count_tokens", "count_tokens",
"ExercisesHelper", "ExercisesHelper",
"LoggerHelper" "LoggerHelper"
] ]

498
app/helpers/exercises.py
View File

@@ -1,249 +1,249 @@
import queue import queue
import random import random
import re import re
import string import string
from wonderwords import RandomWord from wonderwords import RandomWord
from .text import TextHelper from .text import TextHelper
class ExercisesHelper: class ExercisesHelper:
@staticmethod @staticmethod
def divide_number_into_parts(number, parts): def divide_number_into_parts(number, parts):
if number < parts: if number < parts:
return None return None
part_size = number // parts part_size = number // parts
remaining = number % parts remaining = number % parts
q = queue.Queue() q = queue.Queue()
for i in range(parts): for i in range(parts):
if i < remaining: if i < remaining:
q.put(part_size + 1) q.put(part_size + 1)
else: else:
q.put(part_size) q.put(part_size)
return q return q
@staticmethod @staticmethod
def fix_exercise_ids(exercise, start_id): def fix_exercise_ids(exercise, start_id):
# Initialize the starting ID for the first exercise # Initialize the starting ID for the first exercise
current_id = start_id current_id = start_id
questions = exercise["questions"] questions = exercise["questions"]
# Iterate through questions and update the "id" value # Iterate through questions and update the "id" value
for question in questions: for question in questions:
question["id"] = str(current_id) question["id"] = str(current_id)
current_id += 1 current_id += 1
return exercise return exercise
@staticmethod @staticmethod
def replace_first_occurrences_with_placeholders(text: str, words_to_replace: list, start_id): def replace_first_occurrences_with_placeholders(text: str, words_to_replace: list, start_id):
for i, word in enumerate(words_to_replace, start=start_id): for i, word in enumerate(words_to_replace, start=start_id):
# Create a case-insensitive regular expression pattern # Create a case-insensitive regular expression pattern
pattern = re.compile(r'\b' + re.escape(word) + r'\b', re.IGNORECASE) pattern = re.compile(r'\b' + re.escape(word) + r'\b', re.IGNORECASE)
placeholder = '{{' + str(i) + '}}' placeholder = '{{' + str(i) + '}}'
text = pattern.sub(placeholder, text, 1) text = pattern.sub(placeholder, text, 1)
return text return text
@staticmethod @staticmethod
def replace_first_occurrences_with_placeholders_notes(notes: list, words_to_replace: list, start_id): def replace_first_occurrences_with_placeholders_notes(notes: list, words_to_replace: list, start_id):
replaced_notes = [] replaced_notes = []
for i, note in enumerate(notes, start=0): for i, note in enumerate(notes, start=0):
word = words_to_replace[i] word = words_to_replace[i]
pattern = re.compile(r'\b' + re.escape(word) + r'\b', re.IGNORECASE) pattern = re.compile(r'\b' + re.escape(word) + r'\b', re.IGNORECASE)
placeholder = '{{' + str(start_id + i) + '}}' placeholder = '{{' + str(start_id + i) + '}}'
note = pattern.sub(placeholder, note, 1) note = pattern.sub(placeholder, note, 1)
replaced_notes.append(note) replaced_notes.append(note)
return replaced_notes return replaced_notes
@staticmethod @staticmethod
def add_random_words_and_shuffle(word_array, num_random_words): def add_random_words_and_shuffle(word_array, num_random_words):
r = RandomWord() r = RandomWord()
random_words_selected = r.random_words(num_random_words) random_words_selected = r.random_words(num_random_words)
combined_array = word_array + random_words_selected combined_array = word_array + random_words_selected
random.shuffle(combined_array) random.shuffle(combined_array)
result = [] result = []
for i, word in enumerate(combined_array): for i, word in enumerate(combined_array):
letter = chr(65 + i) # chr(65) is 'A' letter = chr(65 + i) # chr(65) is 'A'
result.append({"letter": letter, "word": word}) result.append({"letter": letter, "word": word})
return result return result
@staticmethod @staticmethod
def fillblanks_build_solutions_array(words, start_id): def fillblanks_build_solutions_array(words, start_id):
solutions = [] solutions = []
for i, word in enumerate(words, start=start_id): for i, word in enumerate(words, start=start_id):
solutions.append( solutions.append(
{ {
"id": str(i), "id": str(i),
"solution": word "solution": word
} }
) )
return solutions return solutions
@staticmethod @staticmethod
def remove_excess_questions(questions: [], quantity): def remove_excess_questions(questions: [], quantity):
count_true = 0 count_true = 0
result = [] result = []
for item in reversed(questions): for item in reversed(questions):
if item.get('solution') == 'true' and count_true < quantity: if item.get('solution') == 'true' and count_true < quantity:
count_true += 1 count_true += 1
else: else:
result.append(item) result.append(item)
result.reverse() result.reverse()
return result return result
@staticmethod @staticmethod
def build_write_blanks_text(questions: [], start_id): def build_write_blanks_text(questions: [], start_id):
result = "" result = ""
for i, q in enumerate(questions, start=start_id): for i, q in enumerate(questions, start=start_id):
placeholder = '{{' + str(i) + '}}' placeholder = '{{' + str(i) + '}}'
result = result + q["question"] + placeholder + "\\n" result = result + q["question"] + placeholder + "\\n"
return result return result
@staticmethod @staticmethod
def build_write_blanks_text_form(form: [], start_id): def build_write_blanks_text_form(form: [], start_id):
result = "" result = ""
replaced_words = [] replaced_words = []
for i, entry in enumerate(form, start=start_id): for i, entry in enumerate(form, start=start_id):
placeholder = '{{' + str(i) + '}}' placeholder = '{{' + str(i) + '}}'
# Use regular expression to find the string after ':' # Use regular expression to find the string after ':'
match = re.search(r'(?<=:)\s*(.*)', entry) match = re.search(r'(?<=:)\s*(.*)', entry)
# Extract the matched string # Extract the matched string
original_string = match.group(1) original_string = match.group(1)
# Split the string into words # Split the string into words
words = re.findall(r'\b\w+\b', original_string) words = re.findall(r'\b\w+\b', original_string)
# Remove words with only one letter # Remove words with only one letter
filtered_words = [word for word in words if len(word) > 1] filtered_words = [word for word in words if len(word) > 1]
# Choose a random word from the list of words # Choose a random word from the list of words
selected_word = random.choice(filtered_words) selected_word = random.choice(filtered_words)
pattern = re.compile(r'\b' + re.escape(selected_word) + r'\b', re.IGNORECASE) pattern = re.compile(r'\b' + re.escape(selected_word) + r'\b', re.IGNORECASE)
# Replace the chosen word with the placeholder # Replace the chosen word with the placeholder
replaced_string = pattern.sub(placeholder, original_string, 1) replaced_string = pattern.sub(placeholder, original_string, 1)
# Construct the final replaced string # Construct the final replaced string
replaced_string = entry.replace(original_string, replaced_string) replaced_string = entry.replace(original_string, replaced_string)
result = result + replaced_string + "\\n" result = result + replaced_string + "\\n"
# Save the replaced word or use it as needed # Save the replaced word or use it as needed
# For example, you can save it to a file or a list # For example, you can save it to a file or a list
replaced_words.append(selected_word) replaced_words.append(selected_word)
return result, replaced_words return result, replaced_words
@staticmethod @staticmethod
def build_write_blanks_solutions(questions: [], start_id): def build_write_blanks_solutions(questions: [], start_id):
solutions = [] solutions = []
for i, q in enumerate(questions, start=start_id): for i, q in enumerate(questions, start=start_id):
solution = [q["possible_answers"]] if isinstance(q["possible_answers"], str) else q["possible_answers"] solution = [q["possible_answers"]] if isinstance(q["possible_answers"], str) else q["possible_answers"]
solutions.append( solutions.append(
{ {
"id": str(i), "id": str(i),
"solution": solution "solution": solution
} }
) )
return solutions return solutions
@staticmethod @staticmethod
def build_write_blanks_solutions_listening(words: [], start_id): def build_write_blanks_solutions_listening(words: [], start_id):
solutions = [] solutions = []
for i, word in enumerate(words, start=start_id): for i, word in enumerate(words, start=start_id):
solution = [word] if isinstance(word, str) else word solution = [word] if isinstance(word, str) else word
solutions.append( solutions.append(
{ {
"id": str(i), "id": str(i),
"solution": solution "solution": solution
} }
) )
return solutions return solutions
@staticmethod @staticmethod
def answer_word_limit_ok(question): def answer_word_limit_ok(question):
# Check if any option in any solution has more than three words # Check if any option in any solution has more than three words
return not any( return not any(
len(option.split()) > 3 len(option.split()) > 3
for solution in question["solutions"] for solution in question["solutions"]
for option in solution["solution"] for option in solution["solution"]
) )
@staticmethod @staticmethod
def assign_letters_to_paragraphs(paragraphs): def assign_letters_to_paragraphs(paragraphs):
result = [] result = []
letters = iter(string.ascii_uppercase) letters = iter(string.ascii_uppercase)
for paragraph in paragraphs.split("\n\n"): for paragraph in paragraphs.split("\n\n"):
if TextHelper.has_x_words(paragraph, 10): if TextHelper.has_x_words(paragraph, 10):
result.append({'paragraph': paragraph.strip(), 'letter': next(letters)}) result.append({'paragraph': paragraph.strip(), 'letter': next(letters)})
return result return result
@staticmethod @staticmethod
def contains_empty_dict(arr): def contains_empty_dict(arr):
return any(elem == {} for elem in arr) return any(elem == {} for elem in arr)
@staticmethod @staticmethod
def fix_writing_overall(overall: float, task_response: dict): def fix_writing_overall(overall: float, task_response: dict):
grades = [category["grade"] for category in task_response.values()] grades = [category["grade"] for category in task_response.values()]
if overall > max(grades) or overall < min(grades): if overall > max(grades) or overall < min(grades):
total_sum = sum(grades) total_sum = sum(grades)
average = total_sum / len(grades) average = total_sum / len(grades)
rounded_average = round(average, 0) rounded_average = round(average, 0)
return rounded_average return rounded_average
return overall return overall
@staticmethod @staticmethod
def build_options(ideas): def build_options(ideas):
options = [] options = []
letters = iter(string.ascii_uppercase) letters = iter(string.ascii_uppercase)
for idea in ideas: for idea in ideas:
options.append({ options.append({
"id": next(letters), "id": next(letters),
"sentence": idea["from"] "sentence": idea["from"]
}) })
return options return options
@staticmethod @staticmethod
def build_sentences(ideas, start_id): def build_sentences(ideas, start_id):
sentences = [] sentences = []
letters = iter(string.ascii_uppercase) letters = iter(string.ascii_uppercase)
for idea in ideas: for idea in ideas:
sentences.append({ sentences.append({
"solution": next(letters), "solution": next(letters),
"sentence": idea["idea"] "sentence": idea["idea"]
}) })
random.shuffle(sentences) random.shuffle(sentences)
for i, sentence in enumerate(sentences, start=start_id): for i, sentence in enumerate(sentences, start=start_id):
sentence["id"] = i sentence["id"] = i
return sentences return sentences
@staticmethod @staticmethod
def randomize_mc_options_order(questions): def randomize_mc_options_order(questions):
option_ids = ['A', 'B', 'C', 'D'] option_ids = ['A', 'B', 'C', 'D']
for question in questions: for question in questions:
# Store the original solution text # Store the original solution text
original_solution_text = next( original_solution_text = next(
option['text'] for option in question['options'] if option['id'] == question['solution']) option['text'] for option in question['options'] if option['id'] == question['solution'])
# Shuffle the options # Shuffle the options
random.shuffle(question['options']) random.shuffle(question['options'])
# Update the option ids and find the new solution id # Update the option ids and find the new solution id
for idx, option in enumerate(question['options']): for idx, option in enumerate(question['options']):
option['id'] = option_ids[idx] option['id'] = option_ids[idx]
if option['text'] == original_solution_text: if option['text'] == original_solution_text:
question['solution'] = option['id'] question['solution'] = option['id']
return questions return questions

209
app/helpers/file.py
View File

@@ -1,95 +1,114 @@
import datetime import base64
from pathlib import Path import io
import base64 import os
import io import shutil
import os import subprocess
import shutil import uuid
import subprocess import datetime
from typing import Optional from pathlib import Path
from typing import Optional, Tuple
import numpy as np
import pypandoc import aiofiles
from PIL import Image import numpy as np
import pypandoc
import aiofiles from PIL import Image
class FileHelper: class FileHelper:
@staticmethod @staticmethod
def delete_files_older_than_one_day(directory: str): def delete_files_older_than_one_day(directory: str):
current_time = datetime.datetime.now() current_time = datetime.datetime.now()
for entry in os.scandir(directory): for entry in os.scandir(directory):
if entry.is_file(): if entry.is_file():
file_path = Path(entry) file_path = Path(entry)
file_name = file_path.name file_name = file_path.name
file_modified_time = datetime.datetime.fromtimestamp(file_path.stat().st_mtime) file_modified_time = datetime.datetime.fromtimestamp(file_path.stat().st_mtime)
time_difference = current_time - file_modified_time time_difference = current_time - file_modified_time
if time_difference.days > 1 and "placeholder" not in file_name: if time_difference.days > 1 and "placeholder" not in file_name:
file_path.unlink() file_path.unlink()
print(f"Deleted file: {file_path}") print(f"Deleted file: {file_path}")
# Supposedly pandoc covers a wide range of file extensions only tested with docx # Supposedly pandoc covers a wide range of file extensions only tested with docx
@staticmethod @staticmethod
def convert_file_to_pdf(input_path: str, output_path: str): def convert_file_to_pdf(input_path: str, output_path: str):
pypandoc.convert_file(input_path, 'pdf', outputfile=output_path, extra_args=[ pypandoc.convert_file(input_path, 'pdf', outputfile=output_path, extra_args=[
'-V', 'geometry:paperwidth=5.5in', '-V', 'geometry:paperwidth=5.5in',
'-V', 'geometry:paperheight=8.5in', '-V', 'geometry:paperheight=8.5in',
'-V', 'geometry:margin=0.5in', '-V', 'geometry:margin=0.5in',
'-V', 'pagestyle=empty' '-V', 'pagestyle=empty'
]) ])
@staticmethod @staticmethod
def convert_file_to_html(input_path: str, output_path: str): def convert_file_to_html(input_path: str, output_path: str):
pypandoc.convert_file(input_path, 'html', outputfile=output_path) pypandoc.convert_file(input_path, 'html', outputfile=output_path)
@staticmethod @staticmethod
def pdf_to_png(path_id: str): def pdf_to_png(path_id: str):
to_png = f"pdftoppm -png exercises.pdf page" to_png = f"pdftoppm -png exercises.pdf page"
result = subprocess.run(to_png, shell=True, cwd=f'./tmp/{path_id}', capture_output=True, text=True) result = subprocess.run(to_png, shell=True, cwd=f'./tmp/{path_id}', capture_output=True, text=True)
if result.returncode != 0: if result.returncode != 0:
raise Exception( raise Exception(
f"Couldn't convert pdf to png. Failed to run command '{to_png}' -> ```cmd {result.stderr}```") f"Couldn't convert pdf to png. Failed to run command '{to_png}' -> ```cmd {result.stderr}```")
@staticmethod @staticmethod
def is_page_blank(image_bytes: bytes, image_threshold=10) -> bool: def is_page_blank(image_bytes: bytes, image_threshold=10) -> bool:
with Image.open(io.BytesIO(image_bytes)) as img: with Image.open(io.BytesIO(image_bytes)) as img:
img_gray = img.convert('L') img_gray = img.convert('L')
img_array = np.array(img_gray) img_array = np.array(img_gray)
non_white_pixels = np.sum(img_array < 255) non_white_pixels = np.sum(img_array < 255)
return non_white_pixels <= image_threshold return non_white_pixels <= image_threshold
@classmethod @classmethod
async def _encode_image(cls, image_path: str, image_threshold=10) -> Optional[str]: async def _encode_image(cls, image_path: str, image_threshold=10) -> Optional[str]:
async with aiofiles.open(image_path, "rb") as image_file: async with aiofiles.open(image_path, "rb") as image_file:
image_bytes = await image_file.read() image_bytes = await image_file.read()
if cls.is_page_blank(image_bytes, image_threshold): if cls.is_page_blank(image_bytes, image_threshold):
return None return None
return base64.b64encode(image_bytes).decode('utf-8') return base64.b64encode(image_bytes).decode('utf-8')
@classmethod @classmethod
def b64_pngs(cls, path_id: str, files: list[str]): async def b64_pngs(cls, path_id: str, files: list[str]):
png_messages = [] png_messages = []
for filename in files: for filename in files:
b64_string = cls._encode_image(os.path.join(f'./tmp/{path_id}', filename)) b64_string = await cls._encode_image(os.path.join(f'./tmp/{path_id}', filename))
if b64_string: if b64_string:
png_messages.append({ png_messages.append({
"type": "image_url", "type": "image_url",
"image_url": { "image_url": {
"url": f"data:image/png;base64,{b64_string}" "url": f"data:image/png;base64,{b64_string}"
} }
}) })
return png_messages return png_messages
@staticmethod @staticmethod
def remove_directory(path): def remove_directory(path):
try: try:
if os.path.exists(path): if os.path.exists(path):
if os.path.isdir(path): if os.path.isdir(path):
shutil.rmtree(path) shutil.rmtree(path)
except Exception as e: except Exception as e:
print(f"An error occurred while trying to remove {path}: {str(e)}") print(f"An error occurred while trying to remove {path}: {str(e)}")
@staticmethod
def remove_file(file_path):
try:
if os.path.exists(file_path):
if os.path.isfile(file_path):
os.remove(file_path)
except Exception as e:
print(f"An error occurred while trying to remove the file {file_path}: {str(e)}")
@staticmethod
def save_upload(file) -> Tuple[str, str]:
ext = file.filename.split('.')[-1]
path_id = str(uuid.uuid4())
os.makedirs(f'./tmp/{path_id}', exist_ok=True)
tmp_filename = f'./tmp/{path_id}/uploaded.{ext}'
file.save(tmp_filename)
return ext, path_id

46
app/helpers/logger.py
View File

@@ -1,23 +1,23 @@
import logging import logging
from functools import wraps from functools import wraps
class LoggerHelper: class LoggerHelper:
@staticmethod @staticmethod
def suppress_loggers(): def suppress_loggers():
def decorator(f): def decorator(f):
@wraps(f) @wraps(f)
def wrapped(*args, **kwargs): def wrapped(*args, **kwargs):
root_logger = logging.getLogger() root_logger = logging.getLogger()
original_level = root_logger.level original_level = root_logger.level
root_logger.setLevel(logging.ERROR) root_logger.setLevel(logging.ERROR)
try: try:
return f(*args, **kwargs) return f(*args, **kwargs)
finally: finally:
root_logger.setLevel(original_level) root_logger.setLevel(original_level)
return wrapped return wrapped
return decorator return decorator

56
app/helpers/text.py
View File

@@ -1,28 +1,28 @@
from nltk.corpus import words from nltk.corpus import words
class TextHelper: class TextHelper:
@classmethod @classmethod
def has_words(cls, text: str): def has_words(cls, text: str):
if not cls._has_common_words(text): if not cls._has_common_words(text):
return False return False
english_words = set(words.words()) english_words = set(words.words())
words_in_input = text.split() words_in_input = text.split()
return any(word.lower() in english_words for word in words_in_input) return any(word.lower() in english_words for word in words_in_input)
@classmethod @classmethod
def has_x_words(cls, text: str, quantity): def has_x_words(cls, text: str, quantity):
if not cls._has_common_words(text): if not cls._has_common_words(text):
return False return False
english_words = set(words.words()) english_words = set(words.words())
words_in_input = text.split() words_in_input = text.split()
english_word_count = sum(1 for word in words_in_input if word.lower() in english_words) english_word_count = sum(1 for word in words_in_input if word.lower() in english_words)
return english_word_count >= quantity return english_word_count >= quantity
@staticmethod @staticmethod
def _has_common_words(text: str): def _has_common_words(text: str):
english_words = {"the", "be", "to", "of", "and", "a", "in", "that", "have", "i"} english_words = {"the", "be", "to", "of", "and", "a", "in", "that", "have", "i"}
words_in_input = text.split() words_in_input = text.split()
english_word_count = sum(1 for word in words_in_input if word.lower() in english_words) english_word_count = sum(1 for word in words_in_input if word.lower() in english_words)
return english_word_count >= 10 return english_word_count >= 10

178
app/helpers/token_counter.py
View File

@@ -1,89 +1,89 @@
# This is a work in progress. There are still bugs. Once it is production-ready this will become a full repo. # This is a work in progress. There are still bugs. Once it is production-ready this will become a full repo.
import tiktoken import tiktoken
import nltk import nltk
def count_tokens(text, model_name="gpt-3.5-turbo", debug=False): def count_tokens(text, model_name="gpt-3.5-turbo", debug=False):
""" """
Count the number of tokens in a given text string without using the OpenAI API. Count the number of tokens in a given text string without using the OpenAI API.
This function tries three methods in the following order: This function tries three methods in the following order:
1. tiktoken (preferred): Accurate token counting similar to the OpenAI API. 1. tiktoken (preferred): Accurate token counting similar to the OpenAI API.
2. nltk: Token counting using the Natural Language Toolkit library. 2. nltk: Token counting using the Natural Language Toolkit library.
3. split: Simple whitespace-based token counting as a fallback. 3. split: Simple whitespace-based token counting as a fallback.
Usage: Usage:
------ ------
text = "Your text here" text = "Your text here"
result = count_tokens(text, model_name="gpt-3.5-turbo", debug=True) result = count_tokens(text, model_name="gpt-3.5-turbo", debug=True)
print(result) print(result)
Required libraries: Required libraries:
------------------- -------------------
- tiktoken: Install with 'pip install tiktoken' - tiktoken: Install with 'pip install tiktoken'
- nltk: Install with 'pip install nltk' - nltk: Install with 'pip install nltk'
Parameters: Parameters:
----------- -----------
text : str text : str
The text string for which you want to count tokens. The text string for which you want to count tokens.
model_name : str, optional model_name : str, optional
The OpenAI model for which you want to count tokens (default: "gpt-3.5-turbo"). The OpenAI model for which you want to count tokens (default: "gpt-3.5-turbo").
debug : bool, optional debug : bool, optional
Set to True to print error messages (default: False). Set to True to print error messages (default: False).
Returns: Returns:
-------- --------
result : dict result : dict
A dictionary containing the number of tokens and the method used for counting. A dictionary containing the number of tokens and the method used for counting.
""" """
# Try using tiktoken # Try using tiktoken
try: try:
encoding = tiktoken.encoding_for_model(model_name) encoding = tiktoken.encoding_for_model(model_name)
num_tokens = len(encoding.encode(text)) num_tokens = len(encoding.encode(text))
result = {"n_tokens": num_tokens, "method": "tiktoken"} result = {"n_tokens": num_tokens, "method": "tiktoken"}
return result return result
except Exception as e: except Exception as e:
if debug: if debug:
print(f"Error using tiktoken: {e}") print(f"Error using tiktoken: {e}")
pass pass
# Try using nltk # Try using nltk
try: try:
# Passed nltk.download("punkt") to server.py's @asynccontextmanager # Passed nltk.download("punkt") to server.py's @asynccontextmanager
tokens = nltk.word_tokenize(text) tokens = nltk.word_tokenize(text)
result = {"n_tokens": len(tokens), "method": "nltk"} result = {"n_tokens": len(tokens), "method": "nltk"}
return result return result
except Exception as e: except Exception as e:
if debug: if debug:
print(f"Error using nltk: {e}") print(f"Error using nltk: {e}")
pass pass
# If nltk and tiktoken fail, use a simple split-based method # If nltk and tiktoken fail, use a simple split-based method
tokens = text.split() tokens = text.split()
result = {"n_tokens": len(tokens), "method": "split"} result = {"n_tokens": len(tokens), "method": "split"}
return result return result
class TokenBuffer: class TokenBuffer:
def __init__(self, max_tokens=2048): def __init__(self, max_tokens=2048):
self.max_tokens = max_tokens self.max_tokens = max_tokens
self.buffer = "" self.buffer = ""
self.token_lengths = [] self.token_lengths = []
self.token_count = 0 self.token_count = 0
def update(self, text, model_name="gpt-3.5-turbo", debug=False): def update(self, text, model_name="gpt-3.5-turbo", debug=False):
new_tokens = count_tokens(text, model_name=model_name, debug=debug)["n_tokens"] new_tokens = count_tokens(text, model_name=model_name, debug=debug)["n_tokens"]
self.token_count += new_tokens self.token_count += new_tokens
self.buffer += text self.buffer += text
self.token_lengths.append(new_tokens) self.token_lengths.append(new_tokens)
while self.token_count > self.max_tokens: while self.token_count > self.max_tokens:
removed_tokens = self.token_lengths.pop(0) removed_tokens = self.token_lengths.pop(0)
self.token_count -= removed_tokens self.token_count -= removed_tokens
self.buffer = self.buffer.split(" ", removed_tokens)[-1] self.buffer = self.buffer.split(" ", removed_tokens)[-1]
def get_buffer(self): def get_buffer(self):
return self.buffer return self.buffer

10
app/mappers/__init__.py
View File

@@ -1,5 +1,5 @@
from .exam import ExamMapper from .exam import ExamMapper
__all__ = [ __all__ = [
"ExamMapper" "ExamMapper"
] ]

132
app/mappers/exam.py
View File

@@ -1,66 +1,66 @@
from typing import Dict, Any from typing import Dict, Any
from pydantic import ValidationError from pydantic import ValidationError
from app.dtos.exam import ( from app.dtos.exam import (
MultipleChoiceExercise, MultipleChoiceExercise,
FillBlanksExercise, FillBlanksExercise,
Part, Exam Part, Exam
) )
from app.dtos.sheet import Sheet, Option, MultipleChoiceQuestion, FillBlanksWord from app.dtos.sheet import Sheet, Option, MultipleChoiceQuestion, FillBlanksWord
class ExamMapper: class ExamMapper:
@staticmethod @staticmethod
def map_to_exam_model(response: Dict[str, Any]) -> Exam: def map_to_exam_model(response: Dict[str, Any]) -> Exam:
parts = [] parts = []
for part in response['parts']: for part in response['parts']:
part_exercises = part['exercises'] part_exercises = part['exercises']
context = part.get('context', None) context = part.get('context', None)
exercises = [] exercises = []
for exercise in part_exercises: for exercise in part_exercises:
exercise_type = exercise['type'] exercise_type = exercise['type']
if exercise_type == 'multipleChoice': if exercise_type == 'multipleChoice':
exercise_model = MultipleChoiceExercise(**exercise) exercise_model = MultipleChoiceExercise(**exercise)
elif exercise_type == 'fillBlanks': elif exercise_type == 'fillBlanks':
exercise_model = FillBlanksExercise(**exercise) exercise_model = FillBlanksExercise(**exercise)
else: else:
raise ValidationError(f"Unknown exercise type: {exercise_type}") raise ValidationError(f"Unknown exercise type: {exercise_type}")
exercises.append(exercise_model) exercises.append(exercise_model)
part_kwargs = {"exercises": exercises} part_kwargs = {"exercises": exercises}
if context is not None: if context is not None:
part_kwargs["context"] = context part_kwargs["context"] = context
part_model = Part(**part_kwargs) part_model = Part(**part_kwargs)
parts.append(part_model) parts.append(part_model)
return Exam(parts=parts) return Exam(parts=parts)
@staticmethod @staticmethod
def map_to_sheet(response: Dict[str, Any]) -> Sheet: def map_to_sheet(response: Dict[str, Any]) -> Sheet:
components = [] components = []
for item in response["components"]: for item in response["components"]:
component_type = item["type"] component_type = item["type"]
if component_type == "multipleChoice": if component_type == "multipleChoice":
options = [Option(id=opt["id"], text=opt["text"]) for opt in item["options"]] options = [Option(id=opt["id"], text=opt["text"]) for opt in item["options"]]
components.append(MultipleChoiceQuestion( components.append(MultipleChoiceQuestion(
id=item["id"], id=item["id"],
prompt=item["prompt"], prompt=item["prompt"],
variant=item.get("variant", "text"), variant=item.get("variant", "text"),
options=options options=options
)) ))
elif component_type == "fillBlanks": elif component_type == "fillBlanks":
components.append(FillBlanksWord( components.append(FillBlanksWord(
id=item["id"], id=item["id"],
options=item["options"] options=item["options"]
)) ))
else: else:
components.append(item) components.append(item)
return Sheet(components=components) return Sheet(components=components)

16
app/middlewares/__init__.py
View File

@@ -1,9 +1,9 @@
from .authentication import AuthBackend, AuthenticationMiddleware from .authentication import AuthBackend, AuthenticationMiddleware
from .authorization import Authorized, IsAuthenticatedViaBearerToken from .authorization import Authorized, IsAuthenticatedViaBearerToken
__all__ = [ __all__ = [
"AuthBackend", "AuthBackend",
"AuthenticationMiddleware", "AuthenticationMiddleware",
"Authorized", "Authorized",
"IsAuthenticatedViaBearerToken" "IsAuthenticatedViaBearerToken"
] ]

96
app/middlewares/authentication.py
View File

@@ -1,48 +1,48 @@
import os import os
from typing import Tuple from typing import Tuple
import jwt import jwt
from jwt import InvalidTokenError from jwt import InvalidTokenError
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from starlette.authentication import AuthenticationBackend from starlette.authentication import AuthenticationBackend
from starlette.middleware.authentication import ( from starlette.middleware.authentication import (
AuthenticationMiddleware as BaseAuthenticationMiddleware, AuthenticationMiddleware as BaseAuthenticationMiddleware,
) )
from starlette.requests import HTTPConnection from starlette.requests import HTTPConnection
class Session(BaseModel): class Session(BaseModel):
authenticated: bool = Field(False, description="Is user authenticated?") authenticated: bool = Field(False, description="Is user authenticated?")
class AuthBackend(AuthenticationBackend): class AuthBackend(AuthenticationBackend):
async def authenticate( async def authenticate(
self, conn: HTTPConnection self, conn: HTTPConnection
) -> Tuple[bool, Session]: ) -> Tuple[bool, Session]:
session = Session() session = Session()
authorization: str = conn.headers.get("Authorization") authorization: str = conn.headers.get("Authorization")
if not authorization: if not authorization:
return False, session return False, session
try: try:
scheme, token = authorization.split(" ") scheme, token = authorization.split(" ")
if scheme.lower() != "bearer": if scheme.lower() != "bearer":
return False, session return False, session
except ValueError: except ValueError:
return False, session return False, session
jwt_secret_key = os.getenv("JWT_SECRET_KEY") jwt_secret_key = os.getenv("JWT_SECRET_KEY")
if not jwt_secret_key: if not jwt_secret_key:
return False, session return False, session
try: try:
jwt.decode(token, jwt_secret_key, algorithms=["HS256"]) jwt.decode(token, jwt_secret_key, algorithms=["HS256"])
except InvalidTokenError: except InvalidTokenError:
return False, session return False, session
session.authenticated = True session.authenticated = True
return True, session return True, session
class AuthenticationMiddleware(BaseAuthenticationMiddleware): class AuthenticationMiddleware(BaseAuthenticationMiddleware):
pass pass

72
app/middlewares/authorization.py
View File

@@ -1,36 +1,36 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List, Type from typing import List, Type
from fastapi import Request from fastapi import Request
from fastapi.openapi.models import APIKey, APIKeyIn from fastapi.openapi.models import APIKey, APIKeyIn
from fastapi.security.base import SecurityBase from fastapi.security.base import SecurityBase
from app.exceptions import CustomException, UnauthorizedException from app.exceptions import CustomException, UnauthorizedException
class BaseAuthorization(ABC): class BaseAuthorization(ABC):
exception = CustomException exception = CustomException
@abstractmethod @abstractmethod
async def has_permission(self, request: Request) -> bool: async def has_permission(self, request: Request) -> bool:
pass pass
class IsAuthenticatedViaBearerToken(BaseAuthorization): class IsAuthenticatedViaBearerToken(BaseAuthorization):
exception = UnauthorizedException exception = UnauthorizedException
async def has_permission(self, request: Request) -> bool: async def has_permission(self, request: Request) -> bool:
return request.user.authenticated return request.user.authenticated
class Authorized(SecurityBase): class Authorized(SecurityBase):
def __init__(self, permissions: List[Type[BaseAuthorization]]): def __init__(self, permissions: List[Type[BaseAuthorization]]):
self.permissions = permissions self.permissions = permissions
self.model: APIKey = APIKey(**{"in": APIKeyIn.header}, name="Authorization") self.model: APIKey = APIKey(**{"in": APIKeyIn.header}, name="Authorization")
self.scheme_name = self.__class__.__name__ self.scheme_name = self.__class__.__name__
async def __call__(self, request: Request): async def __call__(self, request: Request):
for permission in self.permissions: for permission in self.permissions:
cls = permission() cls = permission()
if not await cls.has_permission(request=request): if not await cls.has_permission(request=request):
raise cls.exception raise cls.exception

12
app/repositories/abc/__init__.py
View File

@@ -1,7 +1,7 @@
from .file_storage import IFileStorage from .file_storage import IFileStorage
from .document_store import IDocumentStore from .document_store import IDocumentStore
__all__ = [ __all__ = [
"IFileStorage", "IFileStorage",
"IDocumentStore" "IDocumentStore"
] ]

31
app/repositories/abc/document_store.py
View File

@@ -1,16 +1,15 @@
from abc import ABC from abc import ABC
from typing import Dict, Optional, List
class IDocumentStore(ABC):
async def save_to_db(self, collection: str, item): class IDocumentStore(ABC):
pass
async def save_to_db(self, collection: str, item: Dict, doc_id: Optional[str]) -> Optional[str]:
async def save_to_db_with_id(self, collection: str, item, id: str): pass
pass
async def get_all(self, collection: str) -> List[Dict]:
async def get_all(self, collection: str): pass
pass
async def get_doc_by_id(self, collection: str, doc_id: str) -> Optional[Dict]:
async def get_doc_by_id(self, collection: str, doc_id: str): pass
pass

32
app/repositories/abc/file_storage.py
View File

@@ -1,16 +1,16 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IFileStorage(ABC): class IFileStorage(ABC):
@abstractmethod @abstractmethod
async def download_firebase_file(self, source_blob_name, destination_file_name): async def download_firebase_file(self, source_blob_name, destination_file_name):
pass pass
@abstractmethod @abstractmethod
async def upload_file_firebase_get_url(self, destination_blob_name, source_file_name): async def upload_file_firebase_get_url(self, destination_blob_name, source_file_name):
pass pass
@abstractmethod @abstractmethod
async def make_public(self, blob_name: str): async def make_public(self, blob_name: str):
pass pass

16
app/repositories/impl/__init__.py
View File

@@ -1,8 +1,8 @@
from .document_stores import * from .document_stores import *
from .firebase import FirebaseStorage from app.repositories.impl.file_storage.firebase import FirebaseStorage
__all__ = [ __all__ = [
"FirebaseStorage" "FirebaseStorage"
] ]
__all__.extend(document_stores.__all__) __all__.extend(document_stores.__all__)

14
app/repositories/impl/document_stores/__init__.py
View File

@@ -1,7 +1,7 @@
from .firestore import Firestore from .firestore import Firestore
#from .mongo import MongoDB #from .mongo import MongoDB
__all__ = [ __all__ = [
"Firestore", "Firestore",
#"MongoDB" #"MongoDB"
] ]

94
app/repositories/impl/document_stores/firestore.py
View File

@@ -1,47 +1,47 @@
import logging import logging
from google.cloud.firestore_v1.async_client import AsyncClient from typing import Optional, List, Dict
from google.cloud.firestore_v1.async_collection import AsyncCollectionReference
from google.cloud.firestore_v1.async_document import AsyncDocumentReference from google.cloud.firestore_v1.async_client import AsyncClient
from app.repositories.abc import IDocumentStore from google.cloud.firestore_v1.async_collection import AsyncCollectionReference
from google.cloud.firestore_v1.async_document import AsyncDocumentReference
from app.repositories.abc import IDocumentStore
class Firestore(IDocumentStore):
def __init__(self, client: AsyncClient):
self._client = client class Firestore(IDocumentStore):
self._logger = logging.getLogger(__name__) def __init__(self, client: AsyncClient):
self._client = client
async def save_to_db(self, collection: str, item): self._logger = logging.getLogger(__name__)
collection_ref: AsyncCollectionReference = self._client.collection(collection)
update_time, document_ref = await collection_ref.add(item) async def save_to_db(self, collection: str, item, doc_id: Optional[str] = None) -> Optional[str]:
if document_ref: collection_ref: AsyncCollectionReference = self._client.collection(collection)
self._logger.info(f"Document added with ID: {document_ref.id}")
return document_ref.id if doc_id:
else: document_ref: AsyncDocumentReference = collection_ref.document(doc_id)
return None await document_ref.set(item)
doc_snapshot = await document_ref.get()
async def save_to_db_with_id(self, collection: str, item, id: str): if doc_snapshot.exists:
collection_ref: AsyncCollectionReference = self._client.collection(collection) self._logger.info(f"Document added with ID: {document_ref.id}")
document_ref: AsyncDocumentReference = collection_ref.document(id) return document_ref.id
await document_ref.set(item) else:
doc_snapshot = await document_ref.get() update_time, document_ref = await collection_ref.add(item)
if doc_snapshot.exists: if document_ref:
self._logger.info(f"Document added with ID: {document_ref.id}") self._logger.info(f"Document added with ID: {document_ref.id}")
return document_ref.id return document_ref.id
else:
return None return None
async def get_all(self, collection: str): async def get_all(self, collection: str) -> List[Dict]:
collection_ref: AsyncCollectionReference = self._client.collection(collection) collection_ref: AsyncCollectionReference = self._client.collection(collection)
docs = [] docs = []
async for doc in collection_ref.stream(): async for doc in collection_ref.stream():
docs.append(doc.to_dict()) docs.append(doc.to_dict())
return docs return docs
async def get_doc_by_id(self, collection: str, doc_id: str): async def get_doc_by_id(self, collection: str, doc_id: str) -> Optional[Dict]:
collection_ref: AsyncCollectionReference = self._client.collection(collection) collection_ref: AsyncCollectionReference = self._client.collection(collection)
doc_ref: AsyncDocumentReference = collection_ref.document(doc_id) doc_ref: AsyncDocumentReference = collection_ref.document(doc_id)
doc = await doc_ref.get() doc = await doc_ref.get()
if doc.exists: if doc.exists:
return doc.to_dict() return doc.to_dict()
return None return None

73
app/repositories/impl/document_stores/mongo.py
View File

@@ -1,36 +1,37 @@
"""import logging import logging
from pymongo import MongoClient import uuid
from typing import Optional, List, Dict
from app.repositories.abc import IDocumentStore
from motor.motor_asyncio import AsyncIOMotorDatabase
class MongoDB(IDocumentStore): from app.repositories.abc import IDocumentStore
def __init__(self, client: MongoClient):
self._client = client class MongoDB(IDocumentStore):
self._logger = logging.getLogger(__name__)
def __init__(self, mongo_db: AsyncIOMotorDatabase):
def save_to_db(self, collection: str, item): self._mongo_db = mongo_db
collection_ref = self._client[collection] self._logger = logging.getLogger(__name__)
result = collection_ref.insert_one(item)
if result.inserted_id: async def save_to_db(self, collection: str, item, doc_id: Optional[str] = None) -> Optional[str]:
self._logger.info(f"Document added with ID: {result.inserted_id}") collection_ref = self._mongo_db[collection]
return True, str(result.inserted_id)
else: if doc_id is None:
return False, None doc_id = str(uuid.uuid4())
def save_to_db_with_id(self, collection: str, item, doc_id: str): item['id'] = doc_id
collection_ref = self._client[collection]
item['_id'] = doc_id result = await collection_ref.insert_one(item)
result = collection_ref.replace_one({'_id': id}, item, upsert=True) if result.inserted_id:
if result.upserted_id or result.matched_count: # returning id instead of _id
self._logger.info(f"Document added with ID: {doc_id}") self._logger.info(f"Document added with ID: {doc_id}")
return True, doc_id return doc_id
else:
return False, None return None
def get_all(self, collection: str): async def get_all(self, collection: str) -> List[Dict]:
collection_ref = self._client[collection] cursor = self._mongo_db[collection].find()
all_documents = list(collection_ref.find()) return [document async for document in cursor]
return all_documents
""" async def get_doc_by_id(self, collection: str, doc_id: str) -> Optional[Dict]:
return await self._mongo_db[collection].find_one({"id": doc_id})

5
app/repositories/impl/file_storage/__init__.py Normal file
View File

@@ -0,0 +1,5 @@
from .firebase import FirebaseStorage
__all__ = [
"FirebaseStorage"
]

166
app/repositories/impl/firebase.py → app/repositories/impl/file_storage/firebase.py
View File

@@ -1,83 +1,83 @@
import logging import logging
from typing import Optional from typing import Optional
import aiofiles import aiofiles
from httpx import AsyncClient from httpx import AsyncClient
from app.repositories.abc import IFileStorage from app.repositories.abc import IFileStorage
class FirebaseStorage(IFileStorage): class FirebaseStorage(IFileStorage):
def __init__(self, client: AsyncClient, token: str, bucket: str): def __init__(self, client: AsyncClient, token: str, bucket: str):
self._httpx_client = client self._httpx_client = client
self._token = token self._token = token
self._storage_url = f'https://firebasestorage.googleapis.com/v0/b/{bucket}' self._storage_url = f'https://firebasestorage.googleapis.com/v0/b/{bucket}'
self._logger = logging.getLogger(__name__) self._logger = logging.getLogger(__name__)
async def download_firebase_file(self, source_blob_name: str, destination_file_name: str) -> Optional[str]: async def download_firebase_file(self, source_blob_name: str, destination_file_name: str) -> Optional[str]:
source_blob_name = source_blob_name.replace('/', '%2F') source_blob_name = source_blob_name.replace('/', '%2F')
download_url = f"{self._storage_url}/o/{source_blob_name}?alt=media" download_url = f"{self._storage_url}/o/{source_blob_name}?alt=media"
response = await self._httpx_client.get( response = await self._httpx_client.get(
download_url, download_url,
headers={'Authorization': f'Firebase {self._token}'} headers={'Authorization': f'Firebase {self._token}'}
) )
if response.status_code == 200: if response.status_code == 200:
async with aiofiles.open(destination_file_name, 'wb') as file: async with aiofiles.open(destination_file_name, 'wb') as file:
await file.write(response.content) await file.write(response.content)
self._logger.info(f"File downloaded to {destination_file_name}") self._logger.info(f"File downloaded to {destination_file_name}")
return destination_file_name return destination_file_name
else: else:
self._logger.error(f"Failed to download blob {source_blob_name}. {response.status_code} - {response.content}") self._logger.error(f"Failed to download blob {source_blob_name}. {response.status_code} - {response.content}")
return None return None
async def upload_file_firebase_get_url(self, destination_blob_name: str, source_file_name: str) -> Optional[str]: async def upload_file_firebase_get_url(self, destination_blob_name: str, source_file_name: str) -> Optional[str]:
destination_blob_name = destination_blob_name.replace('/', '%2F') destination_blob_name = destination_blob_name.replace('/', '%2F')
upload_url = f"{self._storage_url}/o/{destination_blob_name}" upload_url = f"{self._storage_url}/o/{destination_blob_name}"
async with aiofiles.open(source_file_name, 'rb') as file: async with aiofiles.open(source_file_name, 'rb') as file:
file_bytes = await file.read() file_bytes = await file.read()
response = await self._httpx_client.post( response = await self._httpx_client.post(
upload_url, upload_url,
headers={ headers={
'Authorization': f'Firebase {self._token}', 'Authorization': f'Firebase {self._token}',
"X-Goog-Upload-Protocol": "multipart" "X-Goog-Upload-Protocol": "multipart"
}, },
files={ files={
'metadata': (None, '{"metadata":{"test":"testMetadata"}}', 'application/json'), 'metadata': (None, '{"metadata":{"test":"testMetadata"}}', 'application/json'),
'file': file_bytes 'file': file_bytes
} }
) )
if response.status_code == 200: if response.status_code == 200:
self._logger.info(f"File {source_file_name} uploaded to {self._storage_url}/o/{destination_blob_name}.") self._logger.info(f"File {source_file_name} uploaded to {self._storage_url}/o/{destination_blob_name}.")
# TODO: Test this # TODO: Test this
#await self.make_public(destination_blob_name) #await self.make_public(destination_blob_name)
file_url = f"{self._storage_url}/o/{destination_blob_name}" file_url = f"{self._storage_url}/o/{destination_blob_name}"
return file_url return file_url
else: else:
self._logger.error(f"Failed to upload file {source_file_name}. Error: {response.status_code} - {str(response.content)}") self._logger.error(f"Failed to upload file {source_file_name}. Error: {response.status_code} - {str(response.content)}")
return None return None
async def make_public(self, destination_blob_name: str): async def make_public(self, destination_blob_name: str):
acl_url = f"{self._storage_url}/o/{destination_blob_name}/acl" acl_url = f"{self._storage_url}/o/{destination_blob_name}/acl"
acl = {'entity': 'allUsers', 'role': 'READER'} acl = {'entity': 'allUsers', 'role': 'READER'}
response = await self._httpx_client.post( response = await self._httpx_client.post(
acl_url, acl_url,
headers={ headers={
'Authorization': f'Bearer {self._token}', 'Authorization': f'Bearer {self._token}',
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}, },
json=acl json=acl
) )
if response.status_code == 200: if response.status_code == 200:
self._logger.info(f"Blob {destination_blob_name} is now public.") self._logger.info(f"Blob {destination_blob_name} is now public.")
else: else:
self._logger.error(f"Failed to make blob {destination_blob_name} public. {response.status_code} - {response.content}") self._logger.error(f"Failed to make blob {destination_blob_name} public. {response.status_code} - {response.content}")

316
app/server.py
View File

@@ -1,160 +1,156 @@
import json import json
import os import os
import pathlib import pathlib
import logging.config import logging.config
import logging.handlers import logging.handlers
import aioboto3 import aioboto3
import contextlib import contextlib
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from collections import defaultdict from collections import defaultdict
from typing import List from typing import List
from http import HTTPStatus from http import HTTPStatus
import httpx import httpx
import whisper import whisper
from fastapi import FastAPI, Request from fastapi import FastAPI, Request
from fastapi.encoders import jsonable_encoder from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError from fastapi.exceptions import RequestValidationError
from fastapi.middleware import Middleware from fastapi.middleware import Middleware
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
import nltk import nltk
from dotenv import load_dotenv from starlette import status
from starlette import status
from app.api import router
from app.api import router from app.configs import DependencyInjector
from app.configs import config_di from app.exceptions import CustomException
from app.exceptions import CustomException from app.middlewares import AuthenticationMiddleware, AuthBackend
from app.middlewares import AuthenticationMiddleware, AuthBackend
load_dotenv() @asynccontextmanager
async def lifespan(_app: FastAPI):
"""
@asynccontextmanager Startup and Shutdown logic is in this lifespan method
async def lifespan(_app: FastAPI):
""" https://fastapi.tiangolo.com/advanced/events/
Startup and Shutdown logic is in this lifespan method """
# Whisper model
https://fastapi.tiangolo.com/advanced/events/ whisper_model = whisper.load_model("base")
"""
# Whisper model # NLTK required datasets download
whisper_model = whisper.load_model("base") nltk.download('words')
nltk.download("punkt")
# NLTK required datasets download
nltk.download('words') # AWS Polly client instantiation
nltk.download("punkt") context_stack = contextlib.AsyncExitStack()
session = aioboto3.Session()
# AWS Polly client instantiation polly_client = await context_stack.enter_async_context(
context_stack = contextlib.AsyncExitStack() session.client(
session = aioboto3.Session() 'polly',
polly_client = await context_stack.enter_async_context( region_name='eu-west-1',
session.client( aws_secret_access_key=os.getenv("AWS_ACCESS_KEY_ID"),
'polly', aws_access_key_id=os.getenv("AWS_SECRET_ACCESS_KEY")
region_name='eu-west-1', )
aws_secret_access_key=os.getenv("AWS_ACCESS_KEY_ID"), )
aws_access_key_id=os.getenv("AWS_SECRET_ACCESS_KEY")
) http_client = httpx.AsyncClient()
)
DependencyInjector(
# HTTP Client polly_client,
http_client = httpx.AsyncClient() http_client,
whisper_model
config_di( ).inject()
polly_client=polly_client,
http_client=http_client, # Setup logging
whisper_model=whisper_model config_file = pathlib.Path("./app/configs/logging/logging_config.json")
) with open(config_file) as f_in:
config = json.load(f_in)
# Setup logging
config_file = pathlib.Path("./app/configs/logging/logging_config.json") logging.config.dictConfig(config)
with open(config_file) as f_in:
config = json.load(f_in) yield
logging.config.dictConfig(config) await http_client.aclose()
await polly_client.close()
yield await context_stack.aclose()
await http_client.aclose()
await polly_client.close() def setup_listeners(_app: FastAPI) -> None:
await context_stack.aclose() @_app.exception_handler(RequestValidationError)
async def custom_form_validation_error(request, exc):
"""
def setup_listeners(_app: FastAPI) -> None: Don't delete request param
@_app.exception_handler(RequestValidationError) """
async def custom_form_validation_error(request, exc): reformatted_message = defaultdict(list)
""" for pydantic_error in exc.errors():
Don't delete request param loc, msg = pydantic_error["loc"], pydantic_error["msg"]
""" filtered_loc = loc[1:] if loc[0] in ("body", "query", "path") else loc
reformatted_message = defaultdict(list) field_string = ".".join(filtered_loc)
for pydantic_error in exc.errors(): if field_string == "cookie.refresh_token":
loc, msg = pydantic_error["loc"], pydantic_error["msg"] return JSONResponse(
filtered_loc = loc[1:] if loc[0] in ("body", "query", "path") else loc status_code=401,
field_string = ".".join(filtered_loc) content={"error_code": 401, "message": HTTPStatus.UNAUTHORIZED.description},
if field_string == "cookie.refresh_token": )
return JSONResponse( reformatted_message[field_string].append(msg)
status_code=401,
content={"error_code": 401, "message": HTTPStatus.UNAUTHORIZED.description}, return JSONResponse(
) status_code=status.HTTP_400_BAD_REQUEST,
reformatted_message[field_string].append(msg) content=jsonable_encoder(
{"details": "Invalid request!", "errors": reformatted_message}
return JSONResponse( ),
status_code=status.HTTP_400_BAD_REQUEST, )
content=jsonable_encoder(
{"details": "Invalid request!", "errors": reformatted_message} @_app.exception_handler(CustomException)
), async def custom_exception_handler(request: Request, exc: CustomException):
) """
Don't delete request param
@_app.exception_handler(CustomException) """
async def custom_exception_handler(request: Request, exc: CustomException): return JSONResponse(
""" status_code=exc.code,
Don't delete request param content={"error_code": exc.error_code, "message": exc.message},
""" )
return JSONResponse(
status_code=exc.code, @_app.exception_handler(Exception)
content={"error_code": exc.error_code, "message": exc.message}, async def default_exception_handler(request: Request, exc: Exception):
) """
Don't delete request param
@_app.exception_handler(Exception) """
async def default_exception_handler(request: Request, exc: Exception): return JSONResponse(
""" status_code=500,
Don't delete request param content=str(exc),
""" )
return JSONResponse(
status_code=500,
content=str(exc), def setup_middleware() -> List[Middleware]:
) middleware = [
Middleware(
CORSMiddleware,
def setup_middleware() -> List[Middleware]: allow_origins=["*"],
middleware = [ allow_credentials=True,
Middleware( allow_methods=["*"],
CORSMiddleware, allow_headers=["*"],
allow_origins=["*"], ),
allow_credentials=True, Middleware(
allow_methods=["*"], AuthenticationMiddleware,
allow_headers=["*"], backend=AuthBackend()
), )
Middleware( ]
AuthenticationMiddleware, return middleware
backend=AuthBackend()
)
] def create_app() -> FastAPI:
return middleware env = os.getenv("ENV")
_app = FastAPI(
docs_url="/docs" if env != "production" else None,
def create_app() -> FastAPI: redoc_url="/redoc" if env != "production" else None,
env = os.getenv("ENV") middleware=setup_middleware(),
_app = FastAPI( lifespan=lifespan
docs_url="/docs" if env != "prod" else None, )
redoc_url="/redoc" if env != "prod" else None, _app.include_router(router)
middleware=setup_middleware(), setup_listeners(_app)
lifespan=lifespan return _app
)
_app.include_router(router)
setup_listeners(_app) app = create_app()
return _app
app = create_app()

31
app/services/abc/__init__.py
View File

@@ -1,20 +1,11 @@
from .level import ILevelService from .third_parties import *
from .listening import IListeningService from .exam import *
from .writing import IWritingService from .training import *
from .speaking import ISpeakingService from .user import IUserService
from .reading import IReadingService
from .grade import IGradeService __all__ = [
from .training import ITrainingService "IUserService"
from .kb import IKnowledgeBase ]
from .third_parties import * __all__.extend(third_parties.__all__)
__all__.extend(exam.__all__)
__all__ = [ __all__.extend(training.__all__)
"ILevelService",
"IListeningService",
"IWritingService",
"ISpeakingService",
"IReadingService",
"IGradeService",
"ITrainingService"
]
__all__.extend(third_parties.__all__)

15
app/services/abc/exam/__init__.py Normal file
View File

@@ -0,0 +1,15 @@
from .level import ILevelService
from .listening import IListeningService
from .writing import IWritingService
from .speaking import ISpeakingService
from .reading import IReadingService
from .grade import IGradeService
__all__ = [
"ILevelService",
"IListeningService",
"IWritingService",
"ISpeakingService",
"IReadingService",
"IGradeService",
]

26
app/services/abc/grade.py → app/services/abc/exam/grade.py
View File

@@ -1,13 +1,13 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Dict, List from typing import Dict, List
class IGradeService(ABC): class IGradeService(ABC):
@abstractmethod @abstractmethod
async def grade_short_answers(self, data: Dict): async def grade_short_answers(self, data: Dict):
pass pass
@abstractmethod @abstractmethod
async def calculate_grading_summary(self, extracted_sections: List): async def calculate_grading_summary(self, extracted_sections: List):
pass pass

94
app/services/abc/level.py → app/services/abc/exam/level.py
View File

@@ -1,47 +1,47 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
import random import random
from typing import Dict from typing import Dict
from fastapi import UploadFile from fastapi import UploadFile
from app.configs.constants import EducationalContent from app.configs.constants import EducationalContent
class ILevelService(ABC): class ILevelService(ABC):
@abstractmethod @abstractmethod
async def get_level_exam( async def get_level_exam(
self, number_of_exercises: int = 25, min_timer: int = 25, diagnostic: bool = False self, number_of_exercises: int = 25, min_timer: int = 25, diagnostic: bool = False
) -> Dict: ) -> Dict:
pass pass
@abstractmethod @abstractmethod
async def get_level_utas(self): async def get_level_utas(self):
pass pass
@abstractmethod @abstractmethod
async def get_custom_level(self, data: Dict): async def get_custom_level(self, data: Dict):
pass pass
@abstractmethod @abstractmethod
async def upload_level(self, upload: UploadFile) -> Dict: async def upload_level(self, upload: UploadFile) -> Dict:
pass pass
@abstractmethod @abstractmethod
async def gen_multiple_choice( async def gen_multiple_choice(
self, mc_variant: str, quantity: int, start_id: int = 1, *, utas: bool = False, all_exams=None self, mc_variant: str, quantity: int, start_id: int = 1, *, utas: bool = False, all_exams=None
): ):
pass pass
@abstractmethod @abstractmethod
async def gen_blank_space_text_utas( async def gen_blank_space_text_utas(
self, quantity: int, start_id: int, size: int, topic=random.choice(EducationalContent.MTI_TOPICS) self, quantity: int, start_id: int, size: int, topic=random.choice(EducationalContent.MTI_TOPICS)
): ):
pass pass
@abstractmethod @abstractmethod
async def gen_reading_passage_utas( async def gen_reading_passage_utas(
self, start_id, sa_quantity: int, mc_quantity: int, topic=random.choice(EducationalContent.MTI_TOPICS) self, start_id, sa_quantity: int, mc_quantity: int, topic=random.choice(EducationalContent.MTI_TOPICS)
): ):
pass pass

36
app/services/abc/listening.py → app/services/abc/exam/listening.py
View File

@@ -1,18 +1,18 @@
import queue import queue
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from queue import Queue from queue import Queue
from typing import Dict, List from typing import Dict, List
class IListeningService(ABC): class IListeningService(ABC):
@abstractmethod @abstractmethod
async def get_listening_question( async def get_listening_question(
self, section_id: int, topic: str, req_exercises: List[str], difficulty: str, self, section_id: int, topic: str, req_exercises: List[str], difficulty: str,
number_of_exercises_q=queue.Queue(), start_id=-1 number_of_exercises_q=queue.Queue(), start_id=-1
): ):
pass pass
@abstractmethod @abstractmethod
async def save_listening(self, parts: list[dict], min_timer: int, difficulty: str, listening_id: str) -> Dict: async def save_listening(self, parts: list[dict], min_timer: int, difficulty: str, listening_id: str) -> Dict:
pass pass

44
app/services/abc/reading.py → app/services/abc/exam/reading.py
View File

@@ -1,22 +1,22 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from queue import Queue from queue import Queue
from typing import List from typing import List
class IReadingService(ABC): class IReadingService(ABC):
@abstractmethod @abstractmethod
async def gen_reading_passage( async def gen_reading_passage(
self, self,
passage_id: int, passage_id: int,
topic: str, topic: str,
req_exercises: List[str], req_exercises: List[str],
number_of_exercises_q: Queue, number_of_exercises_q: Queue,
difficulty: str, difficulty: str,
start_id: int start_id: int
): ):
pass pass
@abstractmethod @abstractmethod
async def generate_reading_passage(self, part: int, topic: str, word_count: int = 800): async def generate_reading_passage(self, part: int, topic: str, word_count: int = 800):
pass pass

58
app/services/abc/speaking.py → app/services/abc/exam/speaking.py
View File

@@ -1,29 +1,29 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List, Dict, Optional from typing import List, Dict, Optional
class ISpeakingService(ABC): class ISpeakingService(ABC):
@abstractmethod @abstractmethod
async def get_speaking_part( async def get_speaking_part(
self, part: int, topic: str, difficulty: str, second_topic: Optional[str] = None self, part: int, topic: str, difficulty: str, second_topic: Optional[str] = None
) -> Dict: ) -> Dict:
pass pass
@abstractmethod @abstractmethod
async def grade_speaking_task(self, task: int, answers: List[Dict]) -> Dict: async def grade_speaking_task(self, task: int, answers: List[Dict]) -> Dict:
pass pass
@abstractmethod @abstractmethod
async def create_videos_and_save_to_db(self, exercises: List[Dict], template: Dict, req_id: str): async def create_videos_and_save_to_db(self, exercises: List[Dict], template: Dict, req_id: str):
pass pass
@abstractmethod @abstractmethod
async def generate_video( async def generate_video(
self, part: int, avatar: str, topic: str, questions: list[str], self, part: int, avatar: str, topic: str, questions: list[str],
*, *,
second_topic: Optional[str] = None, second_topic: Optional[str] = None,
prompts: Optional[list[str]] = None, prompts: Optional[list[str]] = None,
suffix: Optional[str] = None, suffix: Optional[str] = None,
): ):
pass pass

22
app/services/abc/writing.py → app/services/abc/exam/writing.py
View File

@@ -1,11 +1,11 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class IWritingService(ABC): class IWritingService(ABC):
@abstractmethod @abstractmethod
async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str):
pass pass
@abstractmethod @abstractmethod
async def grade_writing_task(self, task: int, question: str, answer: str): async def grade_writing_task(self, task: int, question: str, answer: str):
pass pass

26
app/services/abc/third_parties/__init__.py
View File

@@ -1,13 +1,13 @@
from .stt import ISpeechToTextService from .stt import ISpeechToTextService
from .tts import ITextToSpeechService from .tts import ITextToSpeechService
from .llm import ILLMService from .llm import ILLMService
from .vid_gen import IVideoGeneratorService from .vid_gen import IVideoGeneratorService
from .ai_detector import IAIDetectorService from .ai_detector import IAIDetectorService
__all__ = [ __all__ = [
"ISpeechToTextService", "ISpeechToTextService",
"ITextToSpeechService", "ITextToSpeechService",
"ILLMService", "ILLMService",
"IVideoGeneratorService", "IVideoGeneratorService",
"IAIDetectorService" "IAIDetectorService"
] ]

26
app/services/abc/third_parties/ai_detector.py
View File

@@ -1,13 +1,13 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Dict, Optional from typing import Dict, Optional
class IAIDetectorService(ABC): class IAIDetectorService(ABC):
@abstractmethod @abstractmethod
async def run_detection(self, text: str): async def run_detection(self, text: str):
pass pass
@abstractmethod @abstractmethod
def _parse_detection(self, response: Dict) -> Optional[Dict]: def _parse_detection(self, response: Dict) -> Optional[Dict]:
pass pass

76
app/services/abc/third_parties/llm.py
View File

@@ -1,38 +1,38 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List, Optional, TypeVar, Callable from typing import List, Optional, TypeVar, Callable
from openai.types.chat import ChatCompletionMessageParam from openai.types.chat import ChatCompletionMessageParam
from pydantic import BaseModel from pydantic import BaseModel
T = TypeVar('T', bound=BaseModel) T = TypeVar('T', bound=BaseModel)
class ILLMService(ABC): class ILLMService(ABC):
@abstractmethod @abstractmethod
async def prediction( async def prediction(
self, self,
model: str, model: str,
messages: List, messages: List,
fields_to_check: Optional[List[str]], fields_to_check: Optional[List[str]],
temperature: float, temperature: float,
check_blacklisted: bool = True, check_blacklisted: bool = True,
token_count: int = -1 token_count: int = -1
): ):
pass pass
@abstractmethod @abstractmethod
async def prediction_override(self, **kwargs): async def prediction_override(self, **kwargs):
pass pass
@abstractmethod @abstractmethod
async def pydantic_prediction( async def pydantic_prediction(
self, self,
messages: List[ChatCompletionMessageParam], messages: List[ChatCompletionMessageParam],
map_to_model: Callable, map_to_model: Callable,
json_scheme: str, json_scheme: str,
*, *,
model: Optional[str] = None, model: Optional[str] = None,
temperature: Optional[float] = None, temperature: Optional[float] = None,
max_retries: int = 3 max_retries: int = 3
) -> List[T] | T | None: ) -> List[T] | T | None:
pass pass

16
app/services/abc/third_parties/stt.py
View File

@@ -1,8 +1,8 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
class ISpeechToTextService(ABC): class ISpeechToTextService(ABC):
@abstractmethod @abstractmethod
async def speech_to_text(self, file_path): async def speech_to_text(self, file_path):
pass pass

44
app/services/abc/third_parties/tts.py
View File

@@ -1,22 +1,22 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Union from typing import Union
class ITextToSpeechService(ABC): class ITextToSpeechService(ABC):
@abstractmethod @abstractmethod
async def synthesize_speech(self, text: str, voice: str, engine: str, output_format: str): async def synthesize_speech(self, text: str, voice: str, engine: str, output_format: str):
pass pass
@abstractmethod @abstractmethod
async def text_to_speech(self, text: Union[list[str], str], file_name: str): async def text_to_speech(self, text: Union[list[str], str], file_name: str):
pass pass
@abstractmethod @abstractmethod
async def _conversation_to_speech(self, conversation: list): async def _conversation_to_speech(self, conversation: list):
pass pass
@abstractmethod @abstractmethod
async def _text_to_speech(self, text: str): async def _text_to_speech(self, text: str):
pass pass

20
app/services/abc/third_parties/vid_gen.py
View File

@@ -1,10 +1,10 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from app.configs.constants import AvatarEnum from app.configs.constants import AvatarEnum
class IVideoGeneratorService(ABC): class IVideoGeneratorService(ABC):
@abstractmethod @abstractmethod
async def create_video(self, text: str, avatar: str): async def create_video(self, text: str, avatar: str):
pass pass

7
app/services/abc/training/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
from .training import ITrainingService
from .kb import IKnowledgeBase
__all__ = [
"ITrainingService",
"IKnowledgeBase"
]

20
app/services/abc/kb.py → app/services/abc/training/kb.py
View File

@@ -1,10 +1,10 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List, Dict from typing import List, Dict
class IKnowledgeBase(ABC): class IKnowledgeBase(ABC):
@abstractmethod @abstractmethod
def query_knowledge_base(self, query: str, category: str, top_k: int = 5) -> List[Dict[str, str]]: def query_knowledge_base(self, query: str, category: str, top_k: int = 5) -> List[Dict[str, str]]:
pass pass

28
app/services/abc/training.py → app/services/abc/training/training.py
View File

@@ -1,14 +1,14 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Dict from typing import Dict
class ITrainingService(ABC): class ITrainingService(ABC):
@abstractmethod @abstractmethod
async def fetch_tips(self, context: str, question: str, answer: str, correct_answer: str): async def fetch_tips(self, context: str, question: str, answer: str, correct_answer: str):
pass pass
@abstractmethod @abstractmethod
async def get_training_content(self, training_content: Dict) -> Dict: async def get_training_content(self, training_content: Dict) -> Dict:
pass pass

10
app/services/abc/user.py Normal file
View File

@@ -0,0 +1,10 @@
from abc import ABC, abstractmethod
from app.dtos.user_batch import BatchUsersDTO
class IUserService(ABC):
@abstractmethod
async def fetch_tips(self, batch: BatchUsersDTO):
pass

30
app/services/impl/__init__.py
View File

@@ -1,19 +1,11 @@
from .level import LevelService from .user import UserService
from .listening import ListeningService from .training import *
from .reading import ReadingService from .third_parties import *
from .speaking import SpeakingService from .exam import *
from .writing import WritingService
from .grade import GradeService __all__ = [
from .training import * "UserService"
from .third_parties import * ]
__all__.extend(third_parties.__all__)
__all__ = [ __all__.extend(training.__all__)
"LevelService", __all__.extend(exam.__all__)
"ListeningService",
"ReadingService",
"SpeakingService",
"WritingService",
"GradeService",
]
__all__.extend(third_parties.__all__)
__all__.extend(training.__all__)

16
app/services/impl/exam/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
from .level import LevelService
from .listening import ListeningService
from .reading import ReadingService
from .speaking import SpeakingService
from .writing import WritingService
from .grade import GradeService
__all__ = [
"LevelService",
"ListeningService",
"ReadingService",
"SpeakingService",
"WritingService",
"GradeService",
]

400
app/services/impl/grade.py → app/services/impl/exam/grade.py
View File

@@ -1,200 +1,200 @@
import json import json
from typing import List, Dict from typing import List, Dict
from app.configs.constants import GPTModels, TemperatureSettings from app.configs.constants import GPTModels, TemperatureSettings
from app.services.abc import ILLMService, IGradeService from app.services.abc import ILLMService, IGradeService
class GradeService(IGradeService): class GradeService(IGradeService):
def __init__(self, llm: ILLMService): def __init__(self, llm: ILLMService):
self._llm = llm self._llm = llm
async def grade_short_answers(self, data: Dict): async def grade_short_answers(self, data: Dict):
json_format = { json_format = {
"exercises": [ "exercises": [
{ {
"id": 1, "id": 1,
"correct": True, "correct": True,
"correct_answer": " correct answer if wrong" "correct_answer": " correct answer if wrong"
} }
] ]
} }
messages = [ messages = [
{ {
"role": "system", "role": "system",
"content": f'You are a helpful assistant designed to output JSON on this format: {json_format}' "content": f'You are a helpful assistant designed to output JSON on this format: {json_format}'
}, },
{ {
"role": "user", "role": "user",
"content": ( "content": (
'Grade these answers according to the text content and write a correct answer if they are ' 'Grade these answers according to the text content and write a correct answer if they are '
f'wrong. Text, questions and answers:\n {data}' f'wrong. Text, questions and answers:\n {data}'
) )
} }
] ]
return await self._llm.prediction( return await self._llm.prediction(
GPTModels.GPT_4_O, GPTModels.GPT_4_O,
messages, messages,
["exercises"], ["exercises"],
TemperatureSettings.GEN_QUESTION_TEMPERATURE TemperatureSettings.GEN_QUESTION_TEMPERATURE
) )
async def calculate_grading_summary(self, extracted_sections: List): async def calculate_grading_summary(self, extracted_sections: List):
ret = [] ret = []
for section in extracted_sections: for section in extracted_sections:
openai_response_dict = await self._calculate_section_grade_summary(section) openai_response_dict = await self._calculate_section_grade_summary(section)
ret.append( ret.append(
{ {
'code': section['code'], 'code': section['code'],
'name': section['name'], 'name': section['name'],
'grade': section['grade'], 'grade': section['grade'],
'evaluation': openai_response_dict['evaluation'], 'evaluation': openai_response_dict['evaluation'],
'suggestions': openai_response_dict['suggestions'], 'suggestions': openai_response_dict['suggestions'],
'bullet_points': self._parse_bullet_points(openai_response_dict['bullet_points'], section['grade']) 'bullet_points': self._parse_bullet_points(openai_response_dict['bullet_points'], section['grade'])
} }
) )
return {'sections': ret} return {'sections': ret}
async def _calculate_section_grade_summary(self, section): async def _calculate_section_grade_summary(self, section):
section_name = section['name'] section_name = section['name']
section_grade = section['grade'] section_grade = section['grade']
messages = [ messages = [
{ {
"role": "user", "role": "user",
"content": ( "content": (
'You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the ' 'You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the '
'grade obtained in the section. You should offer a evaluation comment on this grade and separately ' 'grade obtained in the section. You should offer a evaluation comment on this grade and separately '
'suggestions on how to possibly get a better grade.' 'suggestions on how to possibly get a better grade.'
) )
}, },
{ {
"role": "user", "role": "user",
"content": f'Section: {str(section_name)} Grade: {str(section_grade)}', "content": f'Section: {str(section_name)} Grade: {str(section_grade)}',
}, },
{ {
"role": "user", "role": "user",
"content": "Speak in third person." "content": "Speak in third person."
}, },
{ {
"role": "user", "role": "user",
"content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section." "content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section."
}, },
{ {
"role": "user", "role": "user",
"content": ( "content": (
"Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, " "Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, "
"be one paragraph long." "be one paragraph long."
) )
}, },
{ {
"role": "user", "role": "user",
"content": "Please save the evaluation comment and suggestions generated." "content": "Please save the evaluation comment and suggestions generated."
}, },
{ {
"role": "user", "role": "user",
"content": f"Offer bullet points to improve the english {str(section_name)} ability." "content": f"Offer bullet points to improve the english {str(section_name)} ability."
}, },
] ]
if section['code'] == "level": if section['code'] == "level":
messages[2:2] = [{ messages[2:2] = [{
"role": "user", "role": "user",
"content": ( "content": (
"This section is comprised of multiple choice questions that measure the user's overall english " "This section is comprised of multiple choice questions that measure the user's overall english "
"level. These multiple choice questions are about knowledge on vocabulary, syntax, grammar rules, " "level. These multiple choice questions are about knowledge on vocabulary, syntax, grammar rules, "
"and contextual usage. The grade obtained measures the ability in these areas and english language " "and contextual usage. The grade obtained measures the ability in these areas and english language "
"overall." "overall."
) )
}] }]
elif section['code'] == "speaking": elif section['code'] == "speaking":
messages[2:2] = [{ messages[2:2] = [{
"role": "user", "role": "user",
"content": ( "content": (
"This section is s designed to assess the English language proficiency of individuals who want to " "This section is s designed to assess the English language proficiency of individuals who want to "
"study or work in English-speaking countries. The speaking section evaluates a candidate's ability " "study or work in English-speaking countries. The speaking section evaluates a candidate's ability "
"to communicate effectively in spoken English." "to communicate effectively in spoken English."
) )
}] }]
chat_config = {'max_tokens': 1000, 'temperature': 0.2} chat_config = {'max_tokens': 1000, 'temperature': 0.2}
tools = self.get_tools() tools = self.get_tools()
res = await self._llm.prediction_override( res = await self._llm.prediction_override(
model="gpt-3.5-turbo", model="gpt-3.5-turbo",
max_tokens=chat_config['max_tokens'], max_tokens=chat_config['max_tokens'],
temperature=chat_config['temperature'], temperature=chat_config['temperature'],
tools=tools, tools=tools,
messages=messages messages=messages
) )
return self._parse_openai_response(res) return self._parse_openai_response(res)
@staticmethod @staticmethod
def _parse_openai_response(response): def _parse_openai_response(response):
if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][ if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][
0] and 'tool_calls' in response['choices'][0]['message'] and isinstance( 0] and 'tool_calls' in response['choices'][0]['message'] and isinstance(
response['choices'][0]['message']['tool_calls'], list) and len( response['choices'][0]['message']['tool_calls'], list) and len(
response['choices'][0]['message']['tool_calls']) > 0 and \ response['choices'][0]['message']['tool_calls']) > 0 and \
response['choices'][0]['message']['tool_calls'][0]['function']['arguments']: response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments']) return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
else: else:
return {'evaluation': "", 'suggestions': "", 'bullet_points': []} return {'evaluation': "", 'suggestions': "", 'bullet_points': []}
@staticmethod @staticmethod
def _parse_bullet_points(bullet_points_str, grade): def _parse_bullet_points(bullet_points_str, grade):
max_grade_for_suggestions = 9 max_grade_for_suggestions = 9
if isinstance(bullet_points_str, str) and grade < max_grade_for_suggestions: if isinstance(bullet_points_str, str) and grade < max_grade_for_suggestions:
# Split the string by '\n' # Split the string by '\n'
lines = bullet_points_str.split('\n') lines = bullet_points_str.split('\n')
# Remove '-' and trim whitespace from each line # Remove '-' and trim whitespace from each line
cleaned_lines = [line.replace('-', '').strip() for line in lines] cleaned_lines = [line.replace('-', '').strip() for line in lines]
# Add '.' to lines that don't end with it # Add '.' to lines that don't end with it
return [line + '.' if line and not line.endswith('.') else line for line in cleaned_lines] return [line + '.' if line and not line.endswith('.') else line for line in cleaned_lines]
else: else:
return [] return []
@staticmethod @staticmethod
def get_tools(): def get_tools():
return [ return [
{ {
"type": "function", "type": "function",
"function": { "function": {
"name": "save_evaluation_and_suggestions", "name": "save_evaluation_and_suggestions",
"description": "Saves the evaluation and suggestions requested by input.", "description": "Saves the evaluation and suggestions requested by input.",
"parameters": { "parameters": {
"type": "object", "type": "object",
"properties": { "properties": {
"evaluation": { "evaluation": {
"type": "string", "type": "string",
"description": ( "description": (
"A comment on the IELTS section grade obtained in the specific section and what " "A comment on the IELTS section grade obtained in the specific section and what "
"it could mean without suggestions." "it could mean without suggestions."
), ),
}, },
"suggestions": { "suggestions": {
"type": "string", "type": "string",
"description": ( "description": (
"A small paragraph text with suggestions on how to possibly get a better grade " "A small paragraph text with suggestions on how to possibly get a better grade "
"than the one obtained." "than the one obtained."
), ),
}, },
"bullet_points": { "bullet_points": {
"type": "string", "type": "string",
"description": ( "description": (
"Text with four bullet points to improve the english speaking ability. Only " "Text with four bullet points to improve the english speaking ability. Only "
"include text for the bullet points separated by a paragraph." "include text for the bullet points separated by a paragraph."
), ),
}, },
}, },
"required": ["evaluation", "suggestions"], "required": ["evaluation", "suggestions"],
}, },
} }
} }
] ]

8
app/services/impl/level/__init__.py → app/services/impl/exam/level/__init__.py
View File

@@ -1,5 +1,5 @@
from .level import LevelService from .level import LevelService
__all__ = [ __all__ = [
"LevelService" "LevelService"
] ]

Some files were not shown because too many files have changed in this diff Show More