import json from typing import List, Dict from app.configs.constants import GPTModels, TemperatureSettings from app.services.abc import ILLMService, IGradeService class GradeService(IGradeService): def __init__(self, llm: ILLMService): self._llm = llm async def grade_short_answers(self, data: Dict): json_format = { "exercises": [ { "id": 1, "correct": True, "correct_answer": " correct answer if wrong" } ] } messages = [ { "role": "system", "content": f'You are a helpful assistant designed to output JSON on this format: {json_format}' }, { "role": "user", "content": ( 'Grade these answers according to the text content and write a correct answer if they are ' f'wrong. Text, questions and answers:\n {data}' ) } ] return await self._llm.prediction( GPTModels.GPT_4_O, messages, ["exercises"], TemperatureSettings.GEN_QUESTION_TEMPERATURE ) async def calculate_grading_summary(self, extracted_sections: List): ret = [] for section in extracted_sections: openai_response_dict = await self._calculate_section_grade_summary(section) ret.append( { 'code': section['code'], 'name': section['name'], 'grade': section['grade'], 'evaluation': openai_response_dict['evaluation'], 'suggestions': openai_response_dict['suggestions'], 'bullet_points': self._parse_bullet_points(openai_response_dict['bullet_points'], section['grade']) } ) return {'sections': ret} async def _calculate_section_grade_summary(self, section): section_name = section['name'] section_grade = section['grade'] messages = [ { "role": "user", "content": ( 'You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the ' 'grade obtained in the section. You should offer a evaluation comment on this grade and separately ' 'suggestions on how to possibly get a better grade.' ) }, { "role": "user", "content": f'Section: {str(section_name)} Grade: {str(section_grade)}', }, { "role": "user", "content": "Speak in third person." }, { "role": "user", "content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section." }, { "role": "user", "content": ( "Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, " "be one paragraph long." ) }, { "role": "user", "content": "Please save the evaluation comment and suggestions generated." }, { "role": "user", "content": f"Offer bullet points to improve the english {str(section_name)} ability." }, ] if section['code'] == "level": messages[2:2] = [{ "role": "user", "content": ( "This section is comprised of multiple choice questions that measure the user's overall english " "level. These multiple choice questions are about knowledge on vocabulary, syntax, grammar rules, " "and contextual usage. The grade obtained measures the ability in these areas and english language " "overall." ) }] elif section['code'] == "speaking": messages[2:2] = [{ "role": "user", "content": ( "This section is s designed to assess the English language proficiency of individuals who want to " "study or work in English-speaking countries. The speaking section evaluates a candidate's ability " "to communicate effectively in spoken English." ) }] chat_config = {'max_tokens': 1000, 'temperature': 0.2} tools = self.get_tools() res = await self._llm.prediction_override( model="gpt-3.5-turbo", max_tokens=chat_config['max_tokens'], temperature=chat_config['temperature'], tools=tools, messages=messages ) return self._parse_openai_response(res) @staticmethod def _parse_openai_response(response): if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][ 0] and 'tool_calls' in response['choices'][0]['message'] and isinstance( response['choices'][0]['message']['tool_calls'], list) and len( response['choices'][0]['message']['tool_calls']) > 0 and \ response['choices'][0]['message']['tool_calls'][0]['function']['arguments']: return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments']) else: return {'evaluation': "", 'suggestions': "", 'bullet_points': []} @staticmethod def _parse_bullet_points(bullet_points_str, grade): max_grade_for_suggestions = 9 if isinstance(bullet_points_str, str) and grade < max_grade_for_suggestions: # Split the string by '\n' lines = bullet_points_str.split('\n') # Remove '-' and trim whitespace from each line cleaned_lines = [line.replace('-', '').strip() for line in lines] # Add '.' to lines that don't end with it return [line + '.' if line and not line.endswith('.') else line for line in cleaned_lines] else: return [] @staticmethod def get_tools(): return [ { "type": "function", "function": { "name": "save_evaluation_and_suggestions", "description": "Saves the evaluation and suggestions requested by input.", "parameters": { "type": "object", "properties": { "evaluation": { "type": "string", "description": ( "A comment on the IELTS section grade obtained in the specific section and what " "it could mean without suggestions." ), }, "suggestions": { "type": "string", "description": ( "A small paragraph text with suggestions on how to possibly get a better grade " "than the one obtained." ), }, "bullet_points": { "type": "string", "description": ( "Text with four bullet points to improve the english speaking ability. Only " "include text for the bullet points separated by a paragraph." ), }, }, "required": ["evaluation", "suggestions"], }, } } ]