encoach_backend/app/services/impl/grade.py

import json
from typing import List
import copy

from app.services.abc import ILLMService, IGradeService


class GradeService(IGradeService):

    chat_config = {'max_tokens': 1000, 'temperature': 0.2}
    tools = [{
        "type": "function",
        "function": {
            "name": "save_evaluation_and_suggestions",
            "description": "Saves the evaluation and suggestions requested by input.",
            "parameters": {
                "type": "object",
                "properties": {
                    "evaluation": {
                        "type": "string",
                        "description": "A comment on the IELTS section grade obtained in the specific section and what it could mean without suggestions.",
                    },
                    "suggestions": {
                        "type": "string",
                        "description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.",
                    },
                    "bullet_points": {
                        "type": "string",
                        "description": "Text with four bullet points to improve the english speaking ability. Only include text for the bullet points separated by a paragraph. ",
                    },
                },
                "required": ["evaluation", "suggestions"],
            },
        }
    }]

    def __init__(self, llm: ILLMService):
        self._llm = llm

    async def calculate_grading_summary(self, extracted_sections: List):
        ret = []

        for section in extracted_sections:
            openai_response_dict = await self._calculate_section_grade_summary(section)
            ret.append(
                {
                    'code': section['code'],
                    'name': section['name'],
                    'grade': section['grade'],
                    'evaluation': openai_response_dict['evaluation'],
                    'suggestions': openai_response_dict['suggestions'],
                    'bullet_points': self._parse_bullet_points(openai_response_dict['bullet_points'], section['grade'])
                }
            )

        return {'sections': ret}

    async def _calculate_section_grade_summary(self, section):
        section_name = section['name']
        section_grade = section['grade']
        messages = [
            {
                "role": "user",
                "content": (
                    'You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the '
                    'grade obtained in the section. You should offer a evaluation comment on this grade and separately '
                    'suggestions on how to possibly get a better grade.'
                )
            },
            {
                "role": "user",
                "content": f'Section: {str(section_name)} Grade: {str(section_grade)}',
            },
            {
                "role": "user",
                "content": "Speak in third person."
            },
            {
                "role": "user",
                "content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section."
            },
            {
                "role": "user",
                "content": (
                    "Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, "
                    "be one paragraph long."
                )
            },
            {
                "role": "user",
                "content": "Please save the evaluation comment and suggestions generated."
            },
            {
                "role": "user",
                "content": f"Offer bullet points to improve the english {str(section_name)} ability."
            },
        ]

        if section['code'] == "level":
            messages[2:2] = [{
                "role": "user",
                "content": (
                    "This section is comprised of multiple choice questions that measure the user's overall english "
                    "level. These multiple choice questions are about knowledge on vocabulary, syntax, grammar rules, "
                    "and contextual usage. The grade obtained measures the ability in these areas and english language "
                    "overall."
                )
            }]
        elif section['code'] == "speaking":
            messages[2:2] = [{
                "role": "user",
                "content": (
                    "This section is s designed to assess the English language proficiency of individuals who want to "
                    "study or work in English-speaking countries. The speaking section evaluates a candidate's ability "
                    "to communicate effectively in spoken English."
                )
            }]

        chat_config = copy.deepcopy(self.chat_config)
        tools = copy.deepcopy(self.tools)

        res = await self._llm.prediction_override(
            model="gpt-3.5-turbo",
            max_tokens=chat_config['max_tokens'],
            temperature=chat_config['temperature'],
            tools=tools,
            messages=messages
        )

        return self._parse_openai_response(res)

    @staticmethod
    def _parse_openai_response(response):
        if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][
            0] and 'tool_calls' in response['choices'][0]['message'] and isinstance(
            response['choices'][0]['message']['tool_calls'], list) and len(
            response['choices'][0]['message']['tool_calls']) > 0 and \
                response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
            return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
        else:
            return {'evaluation': "", 'suggestions': "", 'bullet_points': []}

    @staticmethod
    def _parse_bullet_points(bullet_points_str, grade):
        max_grade_for_suggestions = 9
        if isinstance(bullet_points_str, str) and grade < max_grade_for_suggestions:
            # Split the string by '\n'
            lines = bullet_points_str.split('\n')

            # Remove '-' and trim whitespace from each line
            cleaned_lines = [line.replace('-', '').strip() for line in lines]

            # Add '.' to lines that don't end with it
            return [line + '.' if line and not line.endswith('.') else line for line in cleaned_lines]
        else:
            return []