encoach_backend/app/services/impl/exam/grade.py

import json
from typing import List, Dict

from app.configs.constants import GPTModels, TemperatureSettings
from app.services.abc import ILLMService, IGradeService


class GradeService(IGradeService):

    def __init__(self, llm: ILLMService):
        self._llm = llm

    async def grade_short_answers(self, data: Dict):
        json_format = {
            "exercises": [
                {
                    "id": 1,
                    "correct": True,
                    "correct_answer": " correct answer if wrong"
                }
            ]
        }

        messages = [
            {
                "role": "system",
                "content": f'You are a helpful assistant designed to output JSON on this format: {json_format}'
            },
            {
                "role": "user",
                "content": (
                    'Grade these answers according to the text content and write a correct answer if they are '
                    f'wrong. Text, questions and answers:\n {data}'
                )
            }
        ]

        return await self._llm.prediction(
            GPTModels.GPT_4_O,
            messages,
            ["exercises"],
            TemperatureSettings.GEN_QUESTION_TEMPERATURE
        )

    async def calculate_grading_summary(self, extracted_sections: List):
        ret = []

        for section in extracted_sections:
            openai_response_dict = await self._calculate_section_grade_summary(section)
            ret.append(
                {
                    'code': section['code'],
                    'name': section['name'],
                    'grade': section['grade'],
                    'evaluation': openai_response_dict['evaluation'],
                    'suggestions': openai_response_dict['suggestions'],
                    'bullet_points': self._parse_bullet_points(openai_response_dict['bullet_points'], section['grade'])
                }
            )

        return {'sections': ret}

    async def _calculate_section_grade_summary(self, section):
        section_name = section['name']
        section_grade = section['grade']
        messages = [
            {
                "role": "user",
                "content": (
                    'You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the '
                    'grade obtained in the section. You should offer a evaluation comment on this grade and separately '
                    'suggestions on how to possibly get a better grade.'
                )
            },
            {
                "role": "user",
                "content": f'Section: {str(section_name)} Grade: {str(section_grade)}',
            },
            {
                "role": "user",
                "content": "Speak in third person."
            },
            {
                "role": "user",
                "content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section."
            },
            {
                "role": "user",
                "content": (
                    "Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, "
                    "be one paragraph long."
                )
            },
            {
                "role": "user",
                "content": "Please save the evaluation comment and suggestions generated."
            },
            {
                "role": "user",
                "content": f"Offer bullet points to improve the english {str(section_name)} ability."
            },
        ]

        if section['code'] == "level":
            messages[2:2] = [{
                "role": "user",
                "content": (
                    "This section is comprised of multiple choice questions that measure the user's overall english "
                    "level. These multiple choice questions are about knowledge on vocabulary, syntax, grammar rules, "
                    "and contextual usage. The grade obtained measures the ability in these areas and english language "
                    "overall."
                )
            }]
        elif section['code'] == "speaking":
            messages[2:2] = [{
                "role": "user",
                "content": (
                    "This section is s designed to assess the English language proficiency of individuals who want to "
                    "study or work in English-speaking countries. The speaking section evaluates a candidate's ability "
                    "to communicate effectively in spoken English."
                )
            }]

        chat_config = {'max_tokens': 1000, 'temperature': 0.2}
        tools = self.get_tools()

        res = await self._llm.prediction_override(
            model="gpt-3.5-turbo",
            max_tokens=chat_config['max_tokens'],
            temperature=chat_config['temperature'],
            tools=tools,
            messages=messages
        )

        return self._parse_openai_response(res)

    @staticmethod
    def _parse_openai_response(response):
        if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][
            0] and 'tool_calls' in response['choices'][0]['message'] and isinstance(
            response['choices'][0]['message']['tool_calls'], list) and len(
            response['choices'][0]['message']['tool_calls']) > 0 and \
                response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
            return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
        else:
            return {'evaluation': "", 'suggestions': "", 'bullet_points': []}

    @staticmethod
    def _parse_bullet_points(bullet_points_str, grade):
        max_grade_for_suggestions = 9
        if isinstance(bullet_points_str, str) and grade < max_grade_for_suggestions:
            # Split the string by '\n'
            lines = bullet_points_str.split('\n')

            # Remove '-' and trim whitespace from each line
            cleaned_lines = [line.replace('-', '').strip() for line in lines]

            # Add '.' to lines that don't end with it
            return [line + '.' if line and not line.endswith('.') else line for line in cleaned_lines]
        else:
            return []

    @staticmethod
    def get_tools():
        return [
            {
                "type": "function",
                "function": {
                    "name": "save_evaluation_and_suggestions",
                    "description": "Saves the evaluation and suggestions requested by input.",
                    "parameters": {
                        "type": "object",
                        "properties": {
                            "evaluation": {
                                "type": "string",
                                "description": (
                                    "A comment on the IELTS section grade obtained in the specific section and what "
                                    "it could mean without suggestions."
                                ),
                            },
                            "suggestions": {
                                "type": "string",
                                "description": (
                                    "A small paragraph text with suggestions on how to possibly get a better grade "
                                    "than the one obtained."
                                ),
                            },
                            "bullet_points": {
                                "type": "string",
                                "description": (
                                    "Text with four bullet points to improve the english speaking ability. Only "
                                    "include text for the bullet points separated by a paragraph."
                                ),
                            },
                        },
                        "required": ["evaluation", "suggestions"],
                    },
                }
            }
        ]