14 Commits

Author SHA1 Message Date
Tiago Ribeiro
792502be9a Merged in develop (pull request #59)
Fix speaking self._conf["slides"]["avatar"] giving 'list indices must be integers or slices, not str'
2025-03-05 15:26:50 +00:00
Cristiano Ferreira
9f9d5608dc Fix speaking self._conf["slides"]["avatar"] giving 'list indices must be integers or slices, not str' 2025-03-05 13:44:53 +00:00
Tiago Ribeiro
7212150df6 Merged in develop (pull request #58)
Fix method name
2025-03-04 23:02:38 +00:00
Cristiano Ferreira
b097345c08 Fix method name 2025-03-04 22:38:41 +00:00
Tiago Ribeiro
8144fa49ad Merged in develop (pull request #57)
Develop
2025-03-04 18:30:31 +00:00
Cristiano Ferreira
0c28dd6aee Merged in switch-to-elai (pull request #56)
Switch speaking to use ELAI

Approved-by: Tiago Ribeiro
2025-03-04 16:59:17 +00:00
Cristiano Ferreira
6c156ea876 Switch speaking to use ELAI 2025-03-04 13:32:35 +00:00
carlos.mesquita
7ceade5d40 Merged in release/async (pull request #55)
ENCOA-318 - logger was being used in load_indices_and_metadata before being instantiated

Approved-by: Tiago Ribeiro
2025-03-04 12:23:53 +00:00
Carlos-Mesquita
dceb022baa ENCOA-318 - logger was being used in load_indices_and_metadata before being instantiated 2025-03-04 11:07:09 +00:00
Tiago Ribeiro
39b5d48e67 Merged in develop (pull request #54)
Develop
2025-01-13 22:43:20 +00:00
carlos.mesquita
0c6d07ea68 Merged in release/async (pull request #53)
ENCOA-312

Approved-by: Tiago Ribeiro
2025-01-13 22:42:04 +00:00
Carlos-Mesquita
e1b23ae561 ENCOA-312 2025-01-13 21:03:34 +00:00
carlos.mesquita
e265bc941c Merged in release/async (pull request #52)
ENCOA-311

Approved-by: Tiago Ribeiro
2025-01-13 08:08:30 +00:00
Carlos-Mesquita
b32e38156c ENCOA-311 2025-01-13 01:13:28 +00:00
31 changed files with 189 additions and 79 deletions

View File

@@ -1,4 +1,5 @@
import random import random
from typing import List
from dependency_injector.wiring import Provide, inject from dependency_injector.wiring import Provide, inject
from fastapi import APIRouter, Depends, Path, Query, UploadFile from fastapi import APIRouter, Depends, Path, Query, UploadFile
@@ -31,7 +32,7 @@ async def upload(
@inject @inject
async def generate_listening_dialog( async def generate_listening_dialog(
section: int = Path(..., ge=1, le=4), section: int = Path(..., ge=1, le=4),
difficulty: str = Query(default=None), difficulty: List[str] = Query(default=None),
topic: str = Query(default=None), topic: str = Query(default=None),
listening_controller: IListeningController = Depends(Provide[controller]) listening_controller: IListeningController = Depends(Provide[controller])
): ):

View File

@@ -59,7 +59,7 @@ async def get_speaking_task(
topic: Optional[str] = Query(None), topic: Optional[str] = Query(None),
first_topic: Optional[str] = Query(None), first_topic: Optional[str] = Query(None),
second_topic: Optional[str] = Query(None), second_topic: Optional[str] = Query(None),
difficulty: Optional[str] = None, difficulty: List[str] = Query(default=None),
speaking_controller: ISpeakingController = Depends(Provide[controller]) speaking_controller: ISpeakingController = Depends(Provide[controller])
): ):
if not second_topic: if not second_topic:
@@ -67,8 +67,7 @@ async def get_speaking_task(
else: else:
topic_or_first_topic = first_topic if first_topic else random.choice(EducationalContent.MTI_TOPICS) topic_or_first_topic = first_topic if first_topic else random.choice(EducationalContent.MTI_TOPICS)
if not difficulty: difficulty = [random.choice(EducationalContent.DIFFICULTIES)] if not difficulty else difficulty
difficulty = random.choice(random.choice(EducationalContent.DIFFICULTIES))
second_topic = second_topic if second_topic else random.choice(EducationalContent.MTI_TOPICS) second_topic = second_topic if second_topic else random.choice(EducationalContent.MTI_TOPICS)
return await speaking_controller.get_speaking_part(task, topic_or_first_topic, second_topic, difficulty) return await speaking_controller.get_speaking_part(task, topic_or_first_topic, second_topic, difficulty)

View File

@@ -20,10 +20,10 @@ writing_router = APIRouter()
async def generate_writing_academic( async def generate_writing_academic(
task: int = Path(..., ge=1, le=2), task: int = Path(..., ge=1, le=2),
file: UploadFile = File(...), file: UploadFile = File(...),
difficulty: Optional[List[str]] = None, difficulty: List[str] = Query(default=None),
writing_controller: IWritingController = Depends(Provide[controller]) writing_controller: IWritingController = Depends(Provide[controller])
): ):
difficulty = random.choice(EducationalContent.DIFFICULTIES) if not difficulty else difficulty difficulty = [random.choice(EducationalContent.DIFFICULTIES)] if not difficulty else difficulty
return await writing_controller.get_writing_task_academic_question(task, file, difficulty) return await writing_controller.get_writing_task_academic_question(task, file, difficulty)
@@ -34,10 +34,10 @@ async def generate_writing_academic(
@inject @inject
async def generate_writing( async def generate_writing(
task: int = Path(..., ge=1, le=2), task: int = Path(..., ge=1, le=2),
difficulty: Optional[str] = None, difficulty: List[str] = Query(default=None),
topic: str = Query(default=None), topic: str = Query(default=None),
writing_controller: IWritingController = Depends(Provide[controller]) writing_controller: IWritingController = Depends(Provide[controller])
): ):
difficulty = random.choice(EducationalContent.DIFFICULTIES) if not difficulty else difficulty difficulty = [random.choice(EducationalContent.DIFFICULTIES)] if not difficulty else difficulty
topic = random.choice(EducationalContent.MTI_TOPICS) if not topic else topic topic = random.choice(EducationalContent.MTI_TOPICS) if not topic else topic
return await writing_controller.get_writing_task_general_question(task, topic, difficulty) return await writing_controller.get_writing_task_general_question(task, topic, difficulty)

View File

@@ -51,13 +51,22 @@ class DependencyInjector:
with open('ielts_be/services/impl/third_parties/elai/avatars.json', 'r') as file: with open('ielts_be/services/impl/third_parties/elai/avatars.json', 'r') as file:
elai_avatars = json.load(file) elai_avatars = json.load(file)
with open('ielts_be/services/impl/third_parties/elai/elai_conf.json', 'r') as file:
elai_conf = json.load(file)
""" """
with open('ielts_be/services/impl/third_parties/heygen/avatars.json', 'r') as file: with open('ielts_be/services/impl/third_parties/elai/avatars.json', 'r') as file:
heygen_avatars = json.load(file) elai_avatars = json.load(file)
with open('ielts_be/services/impl/third_parties/elai/conf.json', 'r') as file:
elai_conf = json.load(file)
self._container.vid_gen = providers.Factory( self._container.vid_gen = providers.Factory(
Heygen, client=self._container.http_client, token=os.getenv("HEY_GEN_TOKEN"), avatars=heygen_avatars ELAI, client=self._container.http_client,
token=os.getenv("ELAI_TOKEN"),
avatars=elai_avatars,
conf=elai_conf
) )
self._container.ai_detector = providers.Factory( self._container.ai_detector = providers.Factory(
GPTZero, client=self._container.http_client, gpt_zero_key=os.getenv("GPT_ZERO_API_KEY") GPTZero, client=self._container.http_client, gpt_zero_key=os.getenv("GPT_ZERO_API_KEY")

View File

@@ -1,4 +1,5 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List
from fastapi import UploadFile from fastapi import UploadFile
@@ -10,7 +11,7 @@ class IListeningController(ABC):
pass pass
@abstractmethod @abstractmethod
async def generate_listening_dialog(self, section_id: int, topic: str, difficulty: str): async def generate_listening_dialog(self, section_id: int, topic: str, difficulty: List[str]):
pass pass
@abstractmethod @abstractmethod

View File

@@ -1,10 +1,11 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List
class ISpeakingController(ABC): class ISpeakingController(ABC):
@abstractmethod @abstractmethod
async def get_speaking_part(self, task: int, topic: str, second_topic: str, difficulty: str): async def get_speaking_part(self, task: int, topic: str, second_topic: str, difficulty: List[str]):
pass pass
@abstractmethod @abstractmethod

View File

@@ -1,4 +1,5 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List
from fastapi.datastructures import UploadFile from fastapi.datastructures import UploadFile
@@ -6,9 +7,9 @@ from fastapi.datastructures import UploadFile
class IWritingController(ABC): class IWritingController(ABC):
@abstractmethod @abstractmethod
async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): async def get_writing_task_general_question(self, task: int, topic: str, difficulty: List[str]):
pass pass
@abstractmethod @abstractmethod
async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: str): async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: List[str]):
pass pass

View File

@@ -1,4 +1,5 @@
import io import io
from typing import List
from fastapi import UploadFile from fastapi import UploadFile
from fastapi.responses import StreamingResponse, Response from fastapi.responses import StreamingResponse, Response
@@ -20,7 +21,7 @@ class ListeningController(IListeningController):
else: else:
return res return res
async def generate_listening_dialog(self, section_id: int, topic: str, difficulty: str): async def generate_listening_dialog(self, section_id: int, topic: str, difficulty: List[str]):
return await self._service.generate_listening_dialog(section_id, topic, difficulty) return await self._service.generate_listening_dialog(section_id, topic, difficulty)
async def get_listening_question(self, dto: ListeningExercisesDTO): async def get_listening_question(self, dto: ListeningExercisesDTO):

View File

@@ -1,4 +1,5 @@
import logging import logging
from typing import List
from ielts_be.controllers import ISpeakingController from ielts_be.controllers import ISpeakingController
from ielts_be.services import ISpeakingService, IVideoGeneratorService from ielts_be.services import ISpeakingService, IVideoGeneratorService
@@ -11,7 +12,7 @@ class SpeakingController(ISpeakingController):
self._vid_gen = vid_gen self._vid_gen = vid_gen
self._logger = logging.getLogger(__name__) self._logger = logging.getLogger(__name__)
async def get_speaking_part(self, task: int, topic: str, second_topic: str, difficulty: str): async def get_speaking_part(self, task: int, topic: str, second_topic: str, difficulty: List[str]):
return await self._service.get_speaking_part(task, topic, second_topic, difficulty) return await self._service.get_speaking_part(task, topic, second_topic, difficulty)
async def get_avatars(self): async def get_avatars(self):

View File

@@ -1,3 +1,5 @@
from typing import List
from fastapi import UploadFile, HTTPException from fastapi import UploadFile, HTTPException
from ielts_be.controllers import IWritingController from ielts_be.controllers import IWritingController
@@ -9,10 +11,10 @@ class WritingController(IWritingController):
def __init__(self, writing_service: IWritingService): def __init__(self, writing_service: IWritingService):
self._service = writing_service self._service = writing_service
async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): async def get_writing_task_general_question(self, task: int, topic: str, difficulty: List[str]):
return await self._service.get_writing_task_general_question(task, topic, difficulty) return await self._service.get_writing_task_general_question(task, topic, difficulty)
async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: str): async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: List[str]):
if attachment.content_type not in ['image/jpeg', 'image/png']: if attachment.content_type not in ['image/jpeg', 'image/png']:
raise HTTPException(status_code=400, detail="Invalid file type. Only JPEG and PNG allowed.") raise HTTPException(status_code=400, detail="Invalid file type. Only JPEG and PNG allowed.")
return await self._service.get_writing_task_academic_question(task, attachment, difficulty) return await self._service.get_writing_task_academic_question(task, attachment, difficulty)

View File

@@ -12,7 +12,8 @@ class LevelExercises(BaseModel):
sa_qty: Optional[int] = None sa_qty: Optional[int] = None
mc_qty: Optional[int] = None mc_qty: Optional[int] = None
topic: Optional[str] = None topic: Optional[str] = None
difficulty: Optional[str] = None
class LevelExercisesDTO(BaseModel): class LevelExercisesDTO(BaseModel):
exercises: List[LevelExercises] exercises: List[LevelExercises]
difficulty: Optional[str] = None difficulty: Optional[List[str]] = None

View File

@@ -17,11 +17,12 @@ class SaveListeningDTO(BaseModel):
class ListeningExercises(BaseModel): class ListeningExercises(BaseModel):
type: ListeningExerciseType type: ListeningExerciseType
quantity: int quantity: int
difficulty: Optional[str] = None
class ListeningExercisesDTO(BaseModel): class ListeningExercisesDTO(BaseModel):
text: str text: str
exercises: List[ListeningExercises] exercises: List[ListeningExercises]
difficulty: Optional[str] difficulty: Optional[List[str]] = None
class InstructionsDTO(BaseModel): class InstructionsDTO(BaseModel):
text: str text: str

View File

@@ -10,8 +10,9 @@ class ReadingExercise(BaseModel):
quantity: int quantity: int
num_random_words: Optional[int] = Field(1) num_random_words: Optional[int] = Field(1)
max_words: Optional[int] = Field(3) max_words: Optional[int] = Field(3)
difficulty: Optional[str] = None
class ReadingDTO(BaseModel): class ReadingDTO(BaseModel):
text: str = Field(...) text: str = Field(...)
exercises: List[ReadingExercise] = Field(...) exercises: List[ReadingExercise] = Field(...)
difficulty: Optional[str] = None difficulty: Optional[List[str]] = None

View File

@@ -2,10 +2,12 @@ from .file import FileHelper
from .text import TextHelper from .text import TextHelper
from .token_counter import count_tokens from .token_counter import count_tokens
from .exercises import ExercisesHelper from .exercises import ExercisesHelper
from .difficulty import DifficultyHelper
__all__ = [ __all__ = [
"FileHelper", "FileHelper",
"TextHelper", "TextHelper",
"count_tokens", "count_tokens",
"ExercisesHelper", "ExercisesHelper",
"DifficultyHelper"
] ]

View File

@@ -0,0 +1,40 @@
import math
import random
from typing import Optional, List, Iterator
from ielts_be.configs.constants import EducationalContent
class DifficultyHelper:
def __init__(self, difficulties: Optional[List[str]]):
self.difficulties = difficulties
self.distributed: Optional[Iterator[str]] = None
def distribute_for_count(self, count: int) -> None:
if not self.difficulties or count == 0:
return
result = []
remaining = count
difficulties_count = len(self.difficulties)
for i, diff in enumerate(self.difficulties):
if i == difficulties_count - 1:
slots = remaining
else:
slots = math.ceil(remaining / (difficulties_count - i))
result.extend([diff] * slots)
remaining -= slots
self.distributed = iter(result)
def pick_difficulty(self, difficulty: Optional[str]) -> str:
if difficulty:
return difficulty if difficulty != "Random" else random.choice(EducationalContent.DIFFICULTIES)
if self.distributed:
return next(self.distributed)
return random.choice(EducationalContent.DIFFICULTIES)

View File

@@ -9,7 +9,7 @@ from fastapi import UploadFile
class IListeningService(ABC): class IListeningService(ABC):
@abstractmethod @abstractmethod
async def generate_listening_dialog( self, section_id: int, topic: str, difficulty: str): async def generate_listening_dialog( self, section_id: int, topic: str, difficulty: List[str]):
pass pass
@abstractmethod @abstractmethod

View File

@@ -6,7 +6,7 @@ class ISpeakingService(ABC):
@abstractmethod @abstractmethod
async def get_speaking_part( async def get_speaking_part(
self, part: int, topic: str, second_topic: str, difficulty: str self, part: int, topic: str, second_topic: str, difficulty: List[str]
) -> Dict: ) -> Dict:
pass pass

View File

@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Optional from typing import Optional, List
from fastapi import UploadFile from fastapi import UploadFile
@@ -7,11 +7,11 @@ from fastapi import UploadFile
class IWritingService(ABC): class IWritingService(ABC):
@abstractmethod @abstractmethod
async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): async def get_writing_task_general_question(self, task: int, topic: str, difficulty: List[str]):
pass pass
@abstractmethod @abstractmethod
async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: str): async def get_writing_task_academic_question(self, task: int, attachment: UploadFile, difficulty: List[str]):
pass pass
@abstractmethod @abstractmethod

View File

@@ -8,6 +8,7 @@ import random
from ielts_be.configs.constants import EducationalContent from ielts_be.configs.constants import EducationalContent
from ielts_be.dtos.level import LevelExercisesDTO from ielts_be.dtos.level import LevelExercisesDTO
from ielts_be.helpers import DifficultyHelper
from ielts_be.repositories import IDocumentStore from ielts_be.repositories import IDocumentStore
from ielts_be.services import ( from ielts_be.services import (
ILevelService, ILLMService, IReadingService, ILevelService, ILLMService, IReadingService,
@@ -50,19 +51,21 @@ class LevelService(ILevelService):
async def upload_level(self, upload: UploadFile, solutions: Optional[UploadFile] = None) -> Dict: async def upload_level(self, upload: UploadFile, solutions: Optional[UploadFile] = None) -> Dict:
return await self._upload_module.generate_level_from_file(upload, solutions) return await self._upload_module.generate_level_from_file(upload, solutions)
async def _generate_exercise(self, req_exercise, start_id): async def _generate_exercise(self, req_exercise, start_id, difficulty):
if req_exercise.type == "mcBlank": if req_exercise.type == "mcBlank":
questions = await self._mc.gen_multiple_choice("blank_space", req_exercise.quantity, start_id) questions = await self._mc.gen_multiple_choice("blank_space", req_exercise.quantity, difficulty, start_id)
questions["variant"] = "mcBlank" questions["variant"] = "mcBlank"
questions["type"] = "multipleChoice" questions["type"] = "multipleChoice"
questions["prompt"] = "Choose the correct word or group of words that completes the sentences." questions["prompt"] = "Choose the correct word or group of words that completes the sentences."
questions["difficulty"] = difficulty
return questions return questions
elif req_exercise.type == "mcUnderline": elif req_exercise.type == "mcUnderline":
questions = await self._mc.gen_multiple_choice("underline", req_exercise.quantity, start_id) questions = await self._mc.gen_multiple_choice("underline", req_exercise.quantity, difficulty, start_id)
questions["variant"] = "mcUnderline" questions["variant"] = "mcUnderline"
questions["type"] = "multipleChoice" questions["type"] = "multipleChoice"
questions["prompt"] = "Choose the underlined word or group of words that is not correct." questions["prompt"] = "Choose the underlined word or group of words that is not correct."
questions["difficulty"] = difficulty
return questions return questions
elif req_exercise.type == "passageUtas": elif req_exercise.type == "passageUtas":
@@ -70,34 +73,42 @@ class LevelService(ILevelService):
exercise = await self._passage_utas.gen_reading_passage_utas( exercise = await self._passage_utas.gen_reading_passage_utas(
start_id, start_id,
req_exercise.quantity, req_exercise.quantity,
difficulty,
topic, topic,
req_exercise.text_size req_exercise.text_size
) )
exercise["prompt"] = "Read the text and answer the questions below." exercise["prompt"] = "Read the text and answer the questions below."
exercise["difficulty"] = difficulty
return exercise return exercise
elif req_exercise.type == "fillBlanksMC": elif req_exercise.type == "fillBlanksMC":
exercise = await self._fill_blanks.gen_fill_blanks( exercise = await self._fill_blanks.gen_fill_blanks(
start_id, start_id,
req_exercise.quantity, req_exercise.quantity,
difficulty,
req_exercise.text_size, req_exercise.text_size,
req_exercise.topic req_exercise.topic
) )
exercise["prompt"] = "Read the text below and choose the correct word for each space." exercise["prompt"] = "Read the text below and choose the correct word for each space."
exercise["difficulty"] = difficulty
return exercise return exercise
async def generate_exercises(self, dto: LevelExercisesDTO): async def generate_exercises(self, dto: LevelExercisesDTO):
start_ids = []
current_id = 1 current_id = 1
tasks = []
distributor = DifficultyHelper(dto.difficulty)
none_count = sum(1 for ex in dto.exercises if ex.difficulty is None)
distributor.distribute_for_count(none_count)
for req_exercise in dto.exercises: for req_exercise in dto.exercises:
start_ids.append(current_id) difficulty = distributor.pick_difficulty(req_exercise.difficulty)
tasks.append(
self._generate_exercise(req_exercise, current_id, difficulty)
)
current_id += req_exercise.quantity current_id += req_exercise.quantity
tasks = [
self._generate_exercise(req_exercise, start_id)
for req_exercise, start_id in zip(dto.exercises, start_ids)
]
questions = await gather(*tasks) questions = await gather(*tasks)
questions = [{'id': str(uuid4()), **exercise} for exercise in questions] questions = [{'id': str(uuid4()), **exercise} for exercise in questions]
@@ -105,10 +116,12 @@ class LevelService(ILevelService):
# Just here to support other modules that I don't know if they are supposed to still be used # Just here to support other modules that I don't know if they are supposed to still be used
async def gen_multiple_choice(self, mc_variant: str, quantity: int, start_id: int = 1): async def gen_multiple_choice(self, mc_variant: str, quantity: int, start_id: int = 1):
return await self._mc.gen_multiple_choice(mc_variant, quantity, start_id) difficulty = random.choice(EducationalContent.DIFFICULTIES)
return await self._mc.gen_multiple_choice(mc_variant, quantity, difficulty, start_id)
async def gen_reading_passage_utas(self, start_id, mc_quantity: int, topic=Optional[str]): # sa_quantity: int, async def gen_reading_passage_utas(self, start_id, mc_quantity: int, topic=Optional[str]): # sa_quantity: int,
return await self._passage_utas.gen_reading_passage_utas(start_id, mc_quantity, topic) difficulty = random.choice(EducationalContent.DIFFICULTIES)
return await self._passage_utas.gen_reading_passage_utas(start_id, mc_quantity, difficulty, topic)
async def gen_blank_space_text_utas(self, quantity: int, start_id: int, size: int, topic: str): async def gen_blank_space_text_utas(self, quantity: int, start_id: int, size: int, topic: str):
return await self._blank_space.gen_blank_space_text_utas(quantity, start_id, size, topic) return await self._blank_space.gen_blank_space_text_utas(quantity, start_id, size, topic)

View File

@@ -11,12 +11,10 @@ class FillBlanks:
async def gen_fill_blanks( async def gen_fill_blanks(
self, start_id: int, quantity: int, size: int = 300, topic=None self, start_id: int, quantity: int, difficulty: str, size: int = 300, topic=None
): ):
if not topic: if not topic:
topic = random.choice(EducationalContent.MTI_TOPICS) topic = random.choice(EducationalContent.MTI_TOPICS)
print(quantity)
print(start_id)
messages = [ messages = [
{ {
"role": "system", "role": "system",
@@ -34,8 +32,15 @@ class FillBlanks:
'JSON object containing: the modified text, a solutions array with each word\'s correct ' 'JSON object containing: the modified text, a solutions array with each word\'s correct '
'letter (A-D), and a words array containing each id with four options where one is ' 'letter (A-D), and a words array containing each id with four options where one is '
'the original word (matching the solution) and three are plausible but incorrect ' 'the original word (matching the solution) and three are plausible but incorrect '
'alternatives that maintain grammatical consistency. ' f'alternatives that maintain grammatical consistency and {difficulty} CEFR level complexity. '
'You cannot use repeated words!' #TODO: Solve this after 'You cannot use repeated words!'
# TODO: Solve this after -> forgot about this TODO just saw now in
# 1/11/25 what I meant by this is gpt still sometimes returns repeated
# words even with explicit instructions to not do so, this is a general problem
# for all exercises, for more robust validation use self._llm.pydantic_prediction
# or implement a method that calls a mapper, catches validation error messages
# from that mapper and asks gpt to retry if creating a pydantic model for each
# operation proves to be unsustainable
) )
} }
] ]

View File

@@ -10,16 +10,16 @@ class MultipleChoice:
self._mc_variants = mc_variants self._mc_variants = mc_variants
async def gen_multiple_choice( async def gen_multiple_choice(
self, mc_variant: str, quantity: int, start_id: int = 1 self, mc_variant: str, quantity: int, difficulty: str, start_id: int = 1
): ):
mc_template = self._mc_variants[mc_variant] mc_template = self._mc_variants[mc_variant]
blank_mod = " blank space " if mc_variant == "blank_space" else " " blank_mod = " blank space " if mc_variant == "blank_space" else " "
gen_multiple_choice_for_text: str = ( gen_multiple_choice_for_text: str = (
'Generate {quantity} multiple choice{blank}questions of 4 options for an english level exam, some easy ' 'Generate {quantity} multiple choice{blank}questions of 4 options for an english level exam of {difficulty} '
'questions, some intermediate questions and some advanced questions. Ensure that the questions cover ' 'CEFR level, some easy questions, some intermediate questions and some advanced questions. Ensure that '
'a range of topics such as verb tense, subject-verb agreement, pronoun usage, sentence structure, and ' 'the questions cover a range of topics such as verb tense, subject-verb agreement, pronoun usage, sentence '
'punctuation. Make sure every question only has 1 correct answer.' 'structure, and punctuation. Make sure every question only has 1 correct answer.'
) )
messages = [ messages = [
@@ -31,7 +31,7 @@ class MultipleChoice:
}, },
{ {
"role": "user", "role": "user",
"content": gen_multiple_choice_for_text.format(quantity=str(quantity), blank=blank_mod) "content": gen_multiple_choice_for_text.format(quantity=str(quantity), blank=blank_mod, difficulty=difficulty)
} }
] ]

View File

@@ -13,11 +13,11 @@ class PassageUtas:
self._mc_variants = mc_variants self._mc_variants = mc_variants
async def gen_reading_passage_utas( async def gen_reading_passage_utas(
self, start_id, mc_quantity: int, topic: Optional[str], word_size: Optional[int] # sa_quantity: int, self, start_id, mc_quantity: int, difficulty: str, topic: Optional[str] = None, word_size: Optional[int] = None# sa_quantity: int,
): ):
passage = await self._reading_service.generate_reading_passage(1, topic, word_size) passage = await self._reading_service.generate_reading_passage(1, topic, word_size)
mc_exercises = await self._gen_text_multiple_choice_utas(passage["text"], start_id, mc_quantity) mc_exercises = await self._gen_text_multiple_choice_utas(passage["text"], start_id, mc_quantity, difficulty)
mc_exercises["type"] = "multipleChoice" mc_exercises["type"] = "multipleChoice"
""" """
exercises: { exercises: {
@@ -61,7 +61,7 @@ class PassageUtas:
return question["questions"] return question["questions"]
async def _gen_text_multiple_choice_utas(self, text: str, start_id: int, mc_quantity: int): async def _gen_text_multiple_choice_utas(self, text: str, start_id: int, mc_quantity: int, difficulty: str):
json_template = self._mc_variants["text_mc_utas"] json_template = self._mc_variants["text_mc_utas"]
messages = [ messages = [
@@ -71,7 +71,9 @@ class PassageUtas:
}, },
{ {
"role": "user", "role": "user",
"content": f'Generate {mc_quantity} multiple choice questions of 4 options for this text:\n{text}' "content": (
f'Generate {mc_quantity} multiple choice questions of 4 options, {difficulty} CEFR '
f'level difficulty, for this text:\n{text}')
}, },
{ {
"role": "user", "role": "user",

View File

@@ -1,6 +1,8 @@
import json import json
import random
import uuid import uuid
from ielts_be.configs.constants import EducationalContent
from ielts_be.services import ILLMService from ielts_be.services import ILLMService

View File

@@ -1,7 +1,7 @@
import asyncio import asyncio
from logging import getLogger from logging import getLogger
import random import random
from typing import Dict, Any, Union from typing import Dict, Any, Union, List
from starlette.datastructures import UploadFile from starlette.datastructures import UploadFile
@@ -13,7 +13,7 @@ from ielts_be.configs.constants import (
NeuralVoices, GPTModels, TemperatureSettings, EducationalContent, NeuralVoices, GPTModels, TemperatureSettings, EducationalContent,
FieldsAndExercises FieldsAndExercises
) )
from ielts_be.helpers import FileHelper from ielts_be.helpers import FileHelper, DifficultyHelper
from .audio_to_dialog import AudioToDialog from .audio_to_dialog import AudioToDialog
from .import_listening import ImportListeningModule from .import_listening import ImportListeningModule
from .write_blank_forms import WriteBlankForms from .write_blank_forms import WriteBlankForms
@@ -21,7 +21,6 @@ from .write_blanks import WriteBlanks
from .write_blank_notes import WriteBlankNotes from .write_blank_notes import WriteBlankNotes
from ..shared import TrueFalse, MultipleChoice from ..shared import TrueFalse, MultipleChoice
class ListeningService(IListeningService): class ListeningService(IListeningService):
CONVERSATION_TAIL = ( CONVERSATION_TAIL = (
@@ -94,7 +93,8 @@ class ListeningService(IListeningService):
return await self._import.import_from_file(exercises, solutions) return await self._import.import_from_file(exercises, solutions)
async def generate_listening_dialog(self, section: int, topic: str, difficulty: str): async def generate_listening_dialog(self, section: int, topic: str, difficulty: List[str]):
# TODO: difficulties to difficulty
return await self._sections[f'section_{section}']["generate_dialogue"](section, topic) return await self._sections[f'section_{section}']["generate_dialogue"](section, topic)
async def transcribe_dialog(self, audio: UploadFile): async def transcribe_dialog(self, audio: UploadFile):
@@ -135,6 +135,11 @@ class ListeningService(IListeningService):
start_id = 1 start_id = 1
exercise_tasks = [] exercise_tasks = []
diff_helper = DifficultyHelper(dto.difficulty)
none_count = sum(1 for ex in dto.exercises if ex.difficulty is None)
diff_helper.distribute_for_count(none_count)
for req_exercise in dto.exercises: for req_exercise in dto.exercises:
exercise_tasks.append( exercise_tasks.append(
self._generate_exercise( self._generate_exercise(
@@ -142,7 +147,7 @@ class ListeningService(IListeningService):
"dialog or monologue", "dialog or monologue",
dto.text, dto.text,
start_id, start_id,
dto.difficulty diff_helper.pick_difficulty(req_exercise.difficulty)
) )
) )
start_id += req_exercise.quantity start_id += req_exercise.quantity
@@ -157,6 +162,7 @@ class ListeningService(IListeningService):
question = await self._multiple_choice.gen_multiple_choice( question = await self._multiple_choice.gen_multiple_choice(
text, req_exercise.quantity, start_id, difficulty, n_options text, req_exercise.quantity, start_id, difficulty, n_options
) )
question["difficulty"] = difficulty
self._logger.info(f"Added multiple choice: {question}") self._logger.info(f"Added multiple choice: {question}")
return question return question
@@ -165,6 +171,7 @@ class ListeningService(IListeningService):
dialog_type, text, req_exercise.quantity, start_id, difficulty dialog_type, text, req_exercise.quantity, start_id, difficulty
) )
question["variant"] = "questions" question["variant"] = "questions"
question["difficulty"] = difficulty
self._logger.info(f"Added write blanks questions: {question}") self._logger.info(f"Added write blanks questions: {question}")
return question return question
@@ -173,6 +180,7 @@ class ListeningService(IListeningService):
dialog_type, text, req_exercise.quantity, start_id, difficulty dialog_type, text, req_exercise.quantity, start_id, difficulty
) )
question["variant"] = "fill" question["variant"] = "fill"
question["difficulty"] = difficulty
self._logger.info(f"Added write blanks notes: {question}") self._logger.info(f"Added write blanks notes: {question}")
return question return question
@@ -181,12 +189,14 @@ class ListeningService(IListeningService):
dialog_type, text, req_exercise.quantity, start_id, difficulty dialog_type, text, req_exercise.quantity, start_id, difficulty
) )
question["variant"] = "form" question["variant"] = "form"
question["difficulty"] = difficulty
self._logger.info(f"Added write blanks form: {question}") self._logger.info(f"Added write blanks form: {question}")
return question return question
elif req_exercise.type == "trueFalse": elif req_exercise.type == "trueFalse":
question = await self._true_false.gen_true_false_not_given_exercise( question = await self._true_false.gen_true_false_not_given_exercise(
text, req_exercise.quantity, start_id, difficulty, "listening" text, req_exercise.quantity, start_id, difficulty, "listening"
) )
question["difficulty"] = difficulty
self._logger.info(f"Added trueFalse: {question}") self._logger.info(f"Added trueFalse: {question}")
return question return question

View File

@@ -5,7 +5,7 @@ from fastapi import UploadFile
from ielts_be.configs.constants import GPTModels, FieldsAndExercises, TemperatureSettings from ielts_be.configs.constants import GPTModels, FieldsAndExercises, TemperatureSettings
from ielts_be.dtos.reading import ReadingDTO from ielts_be.dtos.reading import ReadingDTO
from ielts_be.helpers import ExercisesHelper from ielts_be.helpers import ExercisesHelper, DifficultyHelper
from ielts_be.services import IReadingService, ILLMService from ielts_be.services import IReadingService, ILLMService
from .fill_blanks import FillBlanks from .fill_blanks import FillBlanks
from .idea_match import IdeaMatch from .idea_match import IdeaMatch
@@ -84,6 +84,7 @@ class ReadingService(IReadingService):
question = await self._fill_blanks.gen_summary_fill_blanks_exercise( question = await self._fill_blanks.gen_summary_fill_blanks_exercise(
text, req_exercise.quantity, start_id, difficulty, req_exercise.num_random_words text, req_exercise.quantity, start_id, difficulty, req_exercise.num_random_words
) )
question["difficulty"] = difficulty
self._logger.info(f"Added fill blanks: {question}") self._logger.info(f"Added fill blanks: {question}")
return question return question
@@ -91,6 +92,7 @@ class ReadingService(IReadingService):
question = await self._true_false.gen_true_false_not_given_exercise( question = await self._true_false.gen_true_false_not_given_exercise(
text, req_exercise.quantity, start_id, difficulty, "reading" text, req_exercise.quantity, start_id, difficulty, "reading"
) )
question["difficulty"] = difficulty
self._logger.info(f"Added trueFalse: {question}") self._logger.info(f"Added trueFalse: {question}")
return question return question
@@ -100,6 +102,7 @@ class ReadingService(IReadingService):
) )
if ExercisesHelper.answer_word_limit_ok(question): if ExercisesHelper.answer_word_limit_ok(question):
question["difficulty"] = difficulty
self._logger.info(f"Added write blanks: {question}") self._logger.info(f"Added write blanks: {question}")
return question return question
else: else:
@@ -110,6 +113,7 @@ class ReadingService(IReadingService):
question = await self._paragraph_match.gen_paragraph_match_exercise( question = await self._paragraph_match.gen_paragraph_match_exercise(
text, req_exercise.quantity, start_id text, req_exercise.quantity, start_id
) )
question["difficulty"] = difficulty
self._logger.info(f"Added paragraph match: {question}") self._logger.info(f"Added paragraph match: {question}")
return question return question
@@ -118,12 +122,14 @@ class ReadingService(IReadingService):
text, req_exercise.quantity, start_id text, req_exercise.quantity, start_id
) )
question["variant"] = "ideaMatch" question["variant"] = "ideaMatch"
question["difficulty"] = difficulty
self._logger.info(f"Added idea match: {question}") self._logger.info(f"Added idea match: {question}")
return question return question
elif req_exercise.type == "multipleChoice": elif req_exercise.type == "multipleChoice":
question = await self._multiple_choice.gen_multiple_choice( question = await self._multiple_choice.gen_multiple_choice(
text, req_exercise.quantity, start_id, difficulty, 4 text, req_exercise.quantity, start_id, difficulty, 4
) )
question["difficulty"] = difficulty
self._logger.info(f"Added multiple choice: {question}") self._logger.info(f"Added multiple choice: {question}")
return question return question
@@ -131,13 +137,18 @@ class ReadingService(IReadingService):
exercise_tasks = [] exercise_tasks = []
start_id = 1 start_id = 1
diff_helper = DifficultyHelper(dto.difficulty)
none_count = sum(1 for ex in dto.exercises if ex.difficulty is None)
diff_helper.distribute_for_count(none_count)
for req_exercise in dto.exercises: for req_exercise in dto.exercises:
exercise_tasks.append( exercise_tasks.append(
self._generate_single_exercise( self._generate_single_exercise(
req_exercise, req_exercise,
dto.text, dto.text,
start_id, start_id,
dto.difficulty diff_helper.pick_difficulty(req_exercise.difficulty)
) )
) )
start_id += req_exercise.quantity start_id += req_exercise.quantity

View File

@@ -1,5 +1,6 @@
import logging import logging
import re import re
import random
from typing import Dict, List from typing import Dict, List
@@ -99,8 +100,9 @@ class SpeakingService(ISpeakingService):
} }
async def get_speaking_part( async def get_speaking_part(
self, part: int, topic: str, second_topic: str, difficulty: str self, part: int, topic: str, second_topic: str, difficulty: List[str]
) -> Dict: ) -> Dict:
diff = difficulty[0] if len(difficulty) == 1 else random.choice(difficulty)
task_values = self._tasks[f'task_{part}']['get'] task_values = self._tasks[f'task_{part}']['get']
if part == 1: if part == 1:
@@ -157,7 +159,7 @@ class SpeakingService(ISpeakingService):
] ]
response["type"] = part response["type"] = part
response["difficulty"] = difficulty response["difficulty"] = diff
if part in {2, 3}: if part in {2, 3}:
response["topic"] = topic response["topic"] = topic

View File

@@ -1,3 +1,4 @@
import random
from typing import List, Dict, Optional from typing import List, Dict, Optional
from fastapi import UploadFile from fastapi import UploadFile
@@ -16,7 +17,8 @@ class WritingService(IWritingService):
self._llm = llm self._llm = llm
self._grade = GradeWriting(llm, file_storage, ai_detector) self._grade = GradeWriting(llm, file_storage, ai_detector)
async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str): async def get_writing_task_general_question(self, task: int, topic: str, difficulty: List[str]):
diff = difficulty[0] if len(difficulty) == 1 else random.choice(difficulty)
messages = [ messages = [
{ {
"role": "system", "role": "system",
@@ -24,7 +26,7 @@ class WritingService(IWritingService):
'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}' 'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}'
) )
}, },
*get_writing_args_general(task, topic, difficulty) *get_writing_args_general(task, topic, diff)
] ]
llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O
@@ -40,11 +42,12 @@ class WritingService(IWritingService):
return { return {
"question": self._add_newline_before_hyphen(question) if task == 1 else question, "question": self._add_newline_before_hyphen(question) if task == 1 else question,
"difficulty": difficulty, "topic": topic,
"topic": topic "difficulty": diff
} }
async def get_writing_task_academic_question(self, task: int, file: UploadFile, difficulty: str): async def get_writing_task_academic_question(self, task: int, file: UploadFile, difficulty: List[str]):
diff = difficulty[0] if len(difficulty) == 1 else random.choice(difficulty)
messages = [ messages = [
{ {
"role": "system", "role": "system",
@@ -52,7 +55,7 @@ class WritingService(IWritingService):
'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}' 'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}'
) )
}, },
*(await get_writing_args_academic(task, file)) *(await get_writing_args_academic(task, file, diff))
] ]
response = await self._llm.prediction( response = await self._llm.prediction(
@@ -66,7 +69,7 @@ class WritingService(IWritingService):
return { return {
"question": self._add_newline_before_hyphen(question) if task == 1 else question, "question": self._add_newline_before_hyphen(question) if task == 1 else question,
"difficulty": difficulty, "difficulty": diff,
} }
async def grade_writing_task(self, task: int, question: str, answer: str, attachment: Optional[str] = None): async def grade_writing_task(self, task: int, question: str, answer: str, attachment: Optional[str] = None):

View File

@@ -4,7 +4,7 @@ from typing import List, Dict
from fastapi.datastructures import UploadFile from fastapi.datastructures import UploadFile
async def get_writing_args_academic(task: int, attachment: UploadFile) -> List[Dict]: async def get_writing_args_academic(task: int, attachment: UploadFile, difficulty: str) -> List[Dict]:
writing_args = { writing_args = {
"1": { "1": {
"prompt": ( "prompt": (
@@ -16,7 +16,8 @@ async def get_writing_args_academic(task: int, attachment: UploadFile) -> List[D
'The generated prompt must:\n' 'The generated prompt must:\n'
'1. Clearly describe the type of visual representation in the image\n' '1. Clearly describe the type of visual representation in the image\n'
'2. Provide a concise context for the data shown\n' '2. Provide a concise context for the data shown\n'
'3. End with the standard IELTS Task 1 Academic instruction:\n' f'3. Be adequate for {difficulty} CEFR level users\n'
'4. End with the standard IELTS Task 1 Academic instruction:\n'
'"Summarise the information by selecting and reporting the main features, and make comparisons where relevant."' '"Summarise the information by selecting and reporting the main features, and make comparisons where relevant."'
) )
}, },

View File

@@ -35,14 +35,14 @@ class ELAI(IVideoGeneratorService):
voice_provider = self._avatars[avatar].get("voice_provider") voice_provider = self._avatars[avatar].get("voice_provider")
self._conf["slides"][0]["canvas"]["objects"][0]["src"] = avatar_url self._conf["slides"][0]["canvas"]["objects"][0]["src"] = avatar_url
self._conf["slides"]["avatar"] = { self._conf["slides"][0]["avatar"] = {
"code": avatar_code, "code": avatar_code,
"gender": avatar_gender, "gender": avatar_gender,
"canvas": avatar_canvas "canvas": avatar_canvas
} }
self._conf["slides"]["speech"] = text self._conf["slides"][0]["speech"] = text
self._conf["slides"]["voice"] = voice_id self._conf["slides"][0]["voice"] = voice_id
self._conf["slides"]["voiceProvider"] = voice_provider self._conf["slides"][0]["voiceProvider"] = voice_provider
response = await self._http_client.post(self._ELAI_ENDPOINT, headers=self._POST_HEADER, json=self._conf) response = await self._http_client.post(self._ELAI_ENDPOINT, headers=self._POST_HEADER, json=self._conf)
@@ -60,7 +60,7 @@ class ELAI(IVideoGeneratorService):
else: else:
return Task(status=TaskStatus.ERROR) return Task(status=TaskStatus.ERROR)
async def pool_status(self, video_id: str) -> Task: async def poll_status(self, video_id: str) -> Task:
response = await self._http_client.get(f'{self._ELAI_ENDPOINT}/{video_id}', headers=self._GET_HEADER) response = await self._http_client.get(f'{self._ELAI_ENDPOINT}/{video_id}', headers=self._GET_HEADER)
response_data = response.json() response_data = response.json()

View File

@@ -16,8 +16,8 @@ class TrainingContentKnowledgeBase(IKnowledgeBase):
self._tips = None # self._read_json(path) self._tips = None # self._read_json(path)
self._category_metadata = None self._category_metadata = None
self._indices = None self._indices = None
self.load_indices_and_metadata()
self._logger = getLogger(__name__) self._logger = getLogger(__name__)
self.load_indices_and_metadata()
@staticmethod @staticmethod
def _read_json(path: str) -> Dict[str, any]: def _read_json(path: str) -> Dict[str, any]:

View File

@@ -3,5 +3,5 @@ from .logger import suppress_loggers
__all__ = [ __all__ = [
"handle_exception", "handle_exception",
"suppress_loggers" "suppress_loggers",
] ]