Async release
This commit is contained in:
Carlos Mesquita
147
app/services/impl/writing.py
Normal file
147
app/services/impl/writing.py
Normal file
@@ -0,0 +1,147 @@
|
||||
from app.services.abc import IWritingService, ILLMService, IAIDetectorService
|
||||
from app.configs.constants import GPTModels, TemperatureSettings
|
||||
from app.helpers import TextHelper, ExercisesHelper
|
||||
|
||||
|
||||
class WritingService(IWritingService):
|
||||
|
||||
def __init__(self, llm: ILLMService, ai_detector: IAIDetectorService):
|
||||
self._llm = llm
|
||||
self._ai_detector = ai_detector
|
||||
|
||||
async def get_writing_task_general_question(self, task: int, topic: str, difficulty: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}'
|
||||
)
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": self._get_writing_prompt(task, topic, difficulty)
|
||||
}
|
||||
]
|
||||
|
||||
llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O
|
||||
|
||||
response = await self._llm.prediction(
|
||||
llm_model,
|
||||
messages,
|
||||
["prompt"],
|
||||
TemperatureSettings.GEN_QUESTION_TEMPERATURE
|
||||
)
|
||||
|
||||
return {
|
||||
"question": response["prompt"].strip(),
|
||||
"difficulty": difficulty,
|
||||
"topic": topic
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _get_writing_prompt(task: int, topic: str, difficulty: str):
|
||||
return (
|
||||
'Craft a prompt for an IELTS Writing Task 1 General Training exercise that instructs the '
|
||||
'student to compose a letter. The prompt should present a specific scenario or situation, '
|
||||
f'based on the topic of "{topic}", requiring the student to provide information, '
|
||||
'advice, or instructions within the letter. Make sure that the generated prompt is '
|
||||
f'of {difficulty} difficulty and does not contain forbidden subjects in muslim countries.'
|
||||
) if task == 1 else (
|
||||
f'Craft a comprehensive question of {difficulty} difficulty like the ones for IELTS '
|
||||
'Writing Task 2 General Training that directs the candidate to delve into an in-depth '
|
||||
f'analysis of contrasting perspectives on the topic of "{topic}".'
|
||||
)
|
||||
|
||||
async def grade_writing_task(self, task: int, question: str, answer: str):
|
||||
bare_minimum = 100 if task == 1 else 180
|
||||
minimum = 150 if task == 1 else 250
|
||||
|
||||
# TODO: left as is, don't know if this is intended or not
|
||||
llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O
|
||||
temperature = (
|
||||
TemperatureSettings.GRADING_TEMPERATURE
|
||||
if task == 1 else
|
||||
TemperatureSettings.GEN_QUESTION_TEMPERATURE
|
||||
)
|
||||
|
||||
if not TextHelper.has_words(answer):
|
||||
return self._zero_rating("The answer does not contain enough english words.")
|
||||
elif not TextHelper.has_x_words(answer, bare_minimum):
|
||||
return self._zero_rating("The answer is insufficient and too small to be graded.")
|
||||
else:
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
'You are a helpful assistant designed to output JSON on this format: '
|
||||
'{"perfect_answer": "example perfect answer", "comment": '
|
||||
'"comment about answer quality", "overall": 0.0, "task_response": '
|
||||
'{"Task Achievement": 0.0, "Coherence and Cohesion": 0.0, '
|
||||
'"Lexical Resource": 0.0, "Grammatical Range and Accuracy": 0.0 }'
|
||||
)
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f'Evaluate the given Writing Task {task} response based on the IELTS grading system, '
|
||||
'ensuring a strict assessment that penalizes errors. Deduct points for deviations '
|
||||
'from the task, and assign a score of 0 if the response fails to address the question. '
|
||||
f'Additionally, provide an exemplary answer with a minimum of {minimum} words, along with a '
|
||||
'detailed commentary highlighting both strengths and weaknesses in the response. '
|
||||
f'\n Question: "{question}" \n Answer: "{answer}"')
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f'The perfect answer must have at least {minimum} words.'
|
||||
}
|
||||
]
|
||||
|
||||
response = await self._llm.prediction(
|
||||
llm_model,
|
||||
messages,
|
||||
["comment"],
|
||||
temperature
|
||||
)
|
||||
|
||||
response["overall"] = ExercisesHelper.fix_writing_overall(response["overall"], response["task_response"])
|
||||
response['fixed_text'] = await self._get_fixed_text(answer)
|
||||
|
||||
ai_detection = await self._ai_detector.run_detection(answer)
|
||||
if ai_detection is not None:
|
||||
response['ai_detection'] = ai_detection
|
||||
|
||||
return response
|
||||
|
||||
async def _get_fixed_text(self, text):
|
||||
messages = [
|
||||
{"role": "system", "content": ('You are a helpful assistant designed to output JSON on this format: '
|
||||
'{"fixed_text": "fixed test with no misspelling errors"}')
|
||||
},
|
||||
{"role": "user", "content": (
|
||||
'Fix the errors in the given text and put it in a JSON. '
|
||||
f'Do not complete the answer, only replace what is wrong. \n The text: "{text}"')
|
||||
}
|
||||
]
|
||||
|
||||
response = await self._llm.prediction(
|
||||
GPTModels.GPT_3_5_TURBO,
|
||||
messages,
|
||||
["fixed_text"],
|
||||
0.2,
|
||||
False
|
||||
)
|
||||
return response["fixed_text"]
|
||||
|
||||
@staticmethod
|
||||
def _zero_rating(comment: str):
|
||||
return {
|
||||
'comment': comment,
|
||||
'overall': 0,
|
||||
'task_response': {
|
||||
'Coherence and Cohesion': 0,
|
||||
'Grammatical Range and Accuracy': 0,
|
||||
'Lexical Resource': 0,
|
||||
'Task Achievement': 0
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user