Async release
This commit is contained in:
Carlos Mesquita
156
app/services/impl/grade.py
Normal file
156
app/services/impl/grade.py
Normal file
@@ -0,0 +1,156 @@
|
||||
import json
|
||||
from typing import List
|
||||
import copy
|
||||
|
||||
from app.services.abc import ILLMService, IGradeService
|
||||
|
||||
|
||||
class GradeService(IGradeService):
|
||||
|
||||
chat_config = {'max_tokens': 1000, 'temperature': 0.2}
|
||||
tools = [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "save_evaluation_and_suggestions",
|
||||
"description": "Saves the evaluation and suggestions requested by input.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"evaluation": {
|
||||
"type": "string",
|
||||
"description": "A comment on the IELTS section grade obtained in the specific section and what it could mean without suggestions.",
|
||||
},
|
||||
"suggestions": {
|
||||
"type": "string",
|
||||
"description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.",
|
||||
},
|
||||
"bullet_points": {
|
||||
"type": "string",
|
||||
"description": "Text with four bullet points to improve the english speaking ability. Only include text for the bullet points separated by a paragraph. ",
|
||||
},
|
||||
},
|
||||
"required": ["evaluation", "suggestions"],
|
||||
},
|
||||
}
|
||||
}]
|
||||
|
||||
def __init__(self, llm: ILLMService):
|
||||
self._llm = llm
|
||||
|
||||
async def calculate_grading_summary(self, extracted_sections: List):
|
||||
ret = []
|
||||
|
||||
for section in extracted_sections:
|
||||
openai_response_dict = await self._calculate_section_grade_summary(section)
|
||||
ret.append(
|
||||
{
|
||||
'code': section['code'],
|
||||
'name': section['name'],
|
||||
'grade': section['grade'],
|
||||
'evaluation': openai_response_dict['evaluation'],
|
||||
'suggestions': openai_response_dict['suggestions'],
|
||||
'bullet_points': self._parse_bullet_points(openai_response_dict['bullet_points'], section['grade'])
|
||||
}
|
||||
)
|
||||
|
||||
return {'sections': ret}
|
||||
|
||||
async def _calculate_section_grade_summary(self, section):
|
||||
section_name = section['name']
|
||||
section_grade = section['grade']
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
'You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the '
|
||||
'grade obtained in the section. You should offer a evaluation comment on this grade and separately '
|
||||
'suggestions on how to possibly get a better grade.'
|
||||
)
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f'Section: {str(section_name)} Grade: {str(section_grade)}',
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Speak in third person."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, "
|
||||
"be one paragraph long."
|
||||
)
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Please save the evaluation comment and suggestions generated."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"Offer bullet points to improve the english {str(section_name)} ability."
|
||||
},
|
||||
]
|
||||
|
||||
if section['code'] == "level":
|
||||
messages[2:2] = [{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"This section is comprised of multiple choice questions that measure the user's overall english "
|
||||
"level. These multiple choice questions are about knowledge on vocabulary, syntax, grammar rules, "
|
||||
"and contextual usage. The grade obtained measures the ability in these areas and english language "
|
||||
"overall."
|
||||
)
|
||||
}]
|
||||
elif section['code'] == "speaking":
|
||||
messages[2:2] = [{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"This section is s designed to assess the English language proficiency of individuals who want to "
|
||||
"study or work in English-speaking countries. The speaking section evaluates a candidate's ability "
|
||||
"to communicate effectively in spoken English."
|
||||
)
|
||||
}]
|
||||
|
||||
chat_config = copy.deepcopy(self.chat_config)
|
||||
tools = copy.deepcopy(self.tools)
|
||||
|
||||
res = await self._llm.prediction_override(
|
||||
model="gpt-3.5-turbo",
|
||||
max_tokens=chat_config['max_tokens'],
|
||||
temperature=chat_config['temperature'],
|
||||
tools=tools,
|
||||
messages=messages
|
||||
)
|
||||
|
||||
return self._parse_openai_response(res)
|
||||
|
||||
@staticmethod
|
||||
def _parse_openai_response(response):
|
||||
if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][
|
||||
0] and 'tool_calls' in response['choices'][0]['message'] and isinstance(
|
||||
response['choices'][0]['message']['tool_calls'], list) and len(
|
||||
response['choices'][0]['message']['tool_calls']) > 0 and \
|
||||
response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
|
||||
return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
|
||||
else:
|
||||
return {'evaluation': "", 'suggestions': "", 'bullet_points': []}
|
||||
|
||||
@staticmethod
|
||||
def _parse_bullet_points(bullet_points_str, grade):
|
||||
max_grade_for_suggestions = 9
|
||||
if isinstance(bullet_points_str, str) and grade < max_grade_for_suggestions:
|
||||
# Split the string by '\n'
|
||||
lines = bullet_points_str.split('\n')
|
||||
|
||||
# Remove '-' and trim whitespace from each line
|
||||
cleaned_lines = [line.replace('-', '').strip() for line in lines]
|
||||
|
||||
# Add '.' to lines that don't end with it
|
||||
return [line + '.' if line and not line.endswith('.') else line for line in cleaned_lines]
|
||||
else:
|
||||
return []
|
||||
Reference in New Issue
Block a user