devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated this to the latest version of develop, got rid of most of the duplication, might be missing some packages in toml, needs testing

Browse Source
This commit is contained in:
Carlos Mesquita
2024-08-30 02:35:11 +01:00
parent 3cf9fa5cba
commit f92a803d96
73 changed files with 3642 additions and 2703 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
app/services/impl/writing.py
View File

@@ -1,5 +1,7 @@
from typing import List, Dict
from app.services.abc import IWritingService, ILLMService, IAIDetectorService
from app.configs.constants import GPTModels, TemperatureSettings
from app.configs.constants import GPTModels, TemperatureSettings, FieldsAndExercises
from app.helpers import TextHelper, ExercisesHelper
@@ -17,10 +19,7 @@ class WritingService(IWritingService):
'You are a helpful assistant designed to output JSON on this format: {"prompt": "prompt content"}'
)
},
{
"role": "user",
"content": self._get_writing_prompt(task, topic, difficulty)
}
*self._get_writing_messages(task, topic, difficulty)
]
llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O
@@ -32,15 +31,18 @@ class WritingService(IWritingService):
TemperatureSettings.GEN_QUESTION_TEMPERATURE
)
question = response["prompt"].strip()
return {
"question": response["prompt"].strip(),
"question": self._add_newline_before_hyphen(question) if task == 1 else question,
"difficulty": difficulty,
"topic": topic
}
@staticmethod
def _get_writing_prompt(task: int, topic: str, difficulty: str):
return (
def _get_writing_messages(task: int, topic: str, difficulty: str) -> List[Dict]:
# TODO: Should the muslim disclaimer be added to task 2?
task_prompt = (
'Craft a prompt for an IELTS Writing Task 1 General Training exercise that instructs the '
'student to compose a letter. The prompt should present a specific scenario or situation, '
f'based on the topic of "{topic}", requiring the student to provide information, '
@@ -52,32 +54,41 @@ class WritingService(IWritingService):
f'analysis of contrasting perspectives on the topic of "{topic}".'
)
task_instructions = (
'The prompt should end with "In the letter you should" followed by 3 bullet points of what '
'the answer should include.'
) if task == 1 else (
'The question should lead to an answer with either "theories", "complicated information" or '
'be "very descriptive" on the topic.'
)
messages = [
{
"role": "user",
"content": task_prompt
},
{
"role": "user",
"content": task_instructions
}
]
return messages
async def grade_writing_task(self, task: int, question: str, answer: str):
bare_minimum = 100 if task == 1 else 180
minimum = 150 if task == 1 else 250
# TODO: left as is, don't know if this is intended or not
llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O
temperature = (
TemperatureSettings.GRADING_TEMPERATURE
if task == 1 else
TemperatureSettings.GEN_QUESTION_TEMPERATURE
)
if not TextHelper.has_words(answer):
return self._zero_rating("The answer does not contain enough english words.")
elif not TextHelper.has_x_words(answer, bare_minimum):
return self._zero_rating("The answer is insufficient and too small to be graded.")
else:
template = self._get_writing_template()
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"perfect_answer": "example perfect answer", "comment": '
'"comment about answer quality", "overall": 0.0, "task_response": '
'{"Task Achievement": 0.0, "Coherence and Cohesion": 0.0, '
'"Lexical Resource": 0.0, "Grammatical Range and Accuracy": 0.0 }'
f'You are a helpful assistant designed to output JSON on this format: {template}'
)
},
{
@@ -86,16 +97,28 @@ class WritingService(IWritingService):
f'Evaluate the given Writing Task {task} response based on the IELTS grading system, '
'ensuring a strict assessment that penalizes errors. Deduct points for deviations '
'from the task, and assign a score of 0 if the response fails to address the question. '
f'Additionally, provide an exemplary answer with a minimum of {minimum} words, along with a '
'detailed commentary highlighting both strengths and weaknesses in the response. '
'Additionally, provide a detailed commentary highlighting both strengths and '
'weaknesses in the response. '
f'\n Question: "{question}" \n Answer: "{answer}"')
},
{
"role": "user",
"content": f'The perfect answer must have at least {minimum} words.'
}
]
if task == 1:
messages.append({
"role": "user",
"content": (
'Refer to the parts of the letter as: "Greeting Opener", "bullet 1", "bullet 2", '
'"bullet 3", "closer (restate the purpose of the letter)", "closing greeting"'
)
})
llm_model = GPTModels.GPT_3_5_TURBO if task == 1 else GPTModels.GPT_4_O
temperature = (
TemperatureSettings.GRADING_TEMPERATURE
if task == 1 else
TemperatureSettings.GEN_QUESTION_TEMPERATURE
)
response = await self._llm.prediction(
llm_model,
messages,
@@ -103,6 +126,10 @@ class WritingService(IWritingService):
temperature
)
perfect_answer_minimum = 150 if task == 1 else 250
perfect_answer = await self._get_perfect_answer(question, perfect_answer_minimum)
response["perfect_answer"] = perfect_answer["perfect_answer"]
response["overall"] = ExercisesHelper.fix_writing_overall(response["overall"], response["task_response"])
response['fixed_text'] = await self._get_fixed_text(answer)
@@ -114,13 +141,20 @@ class WritingService(IWritingService):
async def _get_fixed_text(self, text):
messages = [
{"role": "system", "content": ('You are a helpful assistant designed to output JSON on this format: '
'{"fixed_text": "fixed test with no misspelling errors"}')
},
{"role": "user", "content": (
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"fixed_text": "fixed test with no misspelling errors"}'
)
},
{
"role": "user",
"content": (
'Fix the errors in the given text and put it in a JSON. '
f'Do not complete the answer, only replace what is wrong. \n The text: "{text}"')
}
f'Do not complete the answer, only replace what is wrong. \n The text: "{text}"'
)
}
]
response = await self._llm.prediction(
@@ -132,16 +166,83 @@ class WritingService(IWritingService):
)
return response["fixed_text"]
async def _get_perfect_answer(self, question: str, size: int) -> Dict:
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"perfect_answer": "perfect answer for the question"}'
)
},
{
"role": "user",
"content": f'Write a perfect answer for this writing exercise of a IELTS exam. Question: {question}'
},
{
"role": "user",
"content": f'The answer must have at least {size} words'
}
]
return await self._llm.prediction(
GPTModels.GPT_4_O,
messages,
["perfect_answer"],
TemperatureSettings.GEN_QUESTION_TEMPERATURE
)
@staticmethod
def _zero_rating(comment: str):
return {
'comment': comment,
'overall': 0,
'task_response': {
'Coherence and Cohesion': 0,
'Grammatical Range and Accuracy': 0,
'Lexical Resource': 0,
'Task Achievement': 0
'Task Achievement': {
"grade": 0.0,
"comment": ""
},
'Coherence and Cohesion': {
"grade": 0.0,
"comment": ""
},
'Lexical Resource': {
"grade": 0.0,
"comment": ""
},
'Grammatical Range and Accuracy': {
"grade": 0.0,
"comment": ""
}
}
}
@staticmethod
def _get_writing_template():
return {
"comment": "comment about student's response quality",
"overall": 0.0,
"task_response": {
"Task Achievement": {
"grade": 0.0,
"comment": "comment about Task Achievement of the student's response"
},
"Coherence and Cohesion": {
"grade": 0.0,
"comment": "comment about Coherence and Cohesion of the student's response"
},
"Lexical Resource": {
"grade": 0.0,
"comment": "comment about Lexical Resource of the student's response"
},
"Grammatical Range and Accuracy": {
"grade": 0.0,
"comment": "comment about Grammatical Range and Accuracy of the student's response"
}
}
}
@staticmethod
def _add_newline_before_hyphen(s):
return s.replace(" -", "\n-")