Fastapi refactor update
This commit is contained in:
Carlos-Mesquita
200
app/services/impl/exam/grade.py
Normal file
200
app/services/impl/exam/grade.py
Normal file
@@ -0,0 +1,200 @@
|
||||
import json
|
||||
from typing import List, Dict
|
||||
|
||||
from app.configs.constants import GPTModels, TemperatureSettings
|
||||
from app.services.abc import ILLMService, IGradeService
|
||||
|
||||
|
||||
class GradeService(IGradeService):
|
||||
|
||||
def __init__(self, llm: ILLMService):
|
||||
self._llm = llm
|
||||
|
||||
async def grade_short_answers(self, data: Dict):
|
||||
json_format = {
|
||||
"exercises": [
|
||||
{
|
||||
"id": 1,
|
||||
"correct": True,
|
||||
"correct_answer": " correct answer if wrong"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": f'You are a helpful assistant designed to output JSON on this format: {json_format}'
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
'Grade these answers according to the text content and write a correct answer if they are '
|
||||
f'wrong. Text, questions and answers:\n {data}'
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
return await self._llm.prediction(
|
||||
GPTModels.GPT_4_O,
|
||||
messages,
|
||||
["exercises"],
|
||||
TemperatureSettings.GEN_QUESTION_TEMPERATURE
|
||||
)
|
||||
|
||||
async def calculate_grading_summary(self, extracted_sections: List):
|
||||
ret = []
|
||||
|
||||
for section in extracted_sections:
|
||||
openai_response_dict = await self._calculate_section_grade_summary(section)
|
||||
ret.append(
|
||||
{
|
||||
'code': section['code'],
|
||||
'name': section['name'],
|
||||
'grade': section['grade'],
|
||||
'evaluation': openai_response_dict['evaluation'],
|
||||
'suggestions': openai_response_dict['suggestions'],
|
||||
'bullet_points': self._parse_bullet_points(openai_response_dict['bullet_points'], section['grade'])
|
||||
}
|
||||
)
|
||||
|
||||
return {'sections': ret}
|
||||
|
||||
async def _calculate_section_grade_summary(self, section):
|
||||
section_name = section['name']
|
||||
section_grade = section['grade']
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
'You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the '
|
||||
'grade obtained in the section. You should offer a evaluation comment on this grade and separately '
|
||||
'suggestions on how to possibly get a better grade.'
|
||||
)
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f'Section: {str(section_name)} Grade: {str(section_grade)}',
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Speak in third person."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, "
|
||||
"be one paragraph long."
|
||||
)
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Please save the evaluation comment and suggestions generated."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"Offer bullet points to improve the english {str(section_name)} ability."
|
||||
},
|
||||
]
|
||||
|
||||
if section['code'] == "level":
|
||||
messages[2:2] = [{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"This section is comprised of multiple choice questions that measure the user's overall english "
|
||||
"level. These multiple choice questions are about knowledge on vocabulary, syntax, grammar rules, "
|
||||
"and contextual usage. The grade obtained measures the ability in these areas and english language "
|
||||
"overall."
|
||||
)
|
||||
}]
|
||||
elif section['code'] == "speaking":
|
||||
messages[2:2] = [{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"This section is s designed to assess the English language proficiency of individuals who want to "
|
||||
"study or work in English-speaking countries. The speaking section evaluates a candidate's ability "
|
||||
"to communicate effectively in spoken English."
|
||||
)
|
||||
}]
|
||||
|
||||
chat_config = {'max_tokens': 1000, 'temperature': 0.2}
|
||||
tools = self.get_tools()
|
||||
|
||||
res = await self._llm.prediction_override(
|
||||
model="gpt-3.5-turbo",
|
||||
max_tokens=chat_config['max_tokens'],
|
||||
temperature=chat_config['temperature'],
|
||||
tools=tools,
|
||||
messages=messages
|
||||
)
|
||||
|
||||
return self._parse_openai_response(res)
|
||||
|
||||
@staticmethod
|
||||
def _parse_openai_response(response):
|
||||
if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][
|
||||
0] and 'tool_calls' in response['choices'][0]['message'] and isinstance(
|
||||
response['choices'][0]['message']['tool_calls'], list) and len(
|
||||
response['choices'][0]['message']['tool_calls']) > 0 and \
|
||||
response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
|
||||
return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
|
||||
else:
|
||||
return {'evaluation': "", 'suggestions': "", 'bullet_points': []}
|
||||
|
||||
@staticmethod
|
||||
def _parse_bullet_points(bullet_points_str, grade):
|
||||
max_grade_for_suggestions = 9
|
||||
if isinstance(bullet_points_str, str) and grade < max_grade_for_suggestions:
|
||||
# Split the string by '\n'
|
||||
lines = bullet_points_str.split('\n')
|
||||
|
||||
# Remove '-' and trim whitespace from each line
|
||||
cleaned_lines = [line.replace('-', '').strip() for line in lines]
|
||||
|
||||
# Add '.' to lines that don't end with it
|
||||
return [line + '.' if line and not line.endswith('.') else line for line in cleaned_lines]
|
||||
else:
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def get_tools():
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "save_evaluation_and_suggestions",
|
||||
"description": "Saves the evaluation and suggestions requested by input.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"evaluation": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"A comment on the IELTS section grade obtained in the specific section and what "
|
||||
"it could mean without suggestions."
|
||||
),
|
||||
},
|
||||
"suggestions": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"A small paragraph text with suggestions on how to possibly get a better grade "
|
||||
"than the one obtained."
|
||||
),
|
||||
},
|
||||
"bullet_points": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Text with four bullet points to improve the english speaking ability. Only "
|
||||
"include text for the bullet points separated by a paragraph."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["evaluation", "suggestions"],
|
||||
},
|
||||
}
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user