devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve speaking grading.

Browse Source
This commit is contained in:
Cristiano Ferreira
2023-09-15 00:01:52 +01:00
parent 8f9b65281e
commit f77fafa864
3 changed files with 67 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app.py
View File

@@ -8,7 +8,7 @@ from helper.file_helper import delete_files_older_than_one_day
from helper.firebase_helper import download_firebase_file, upload_file_firebase, upload_file_firebase_get_url, \
save_to_db
from helper.heygen_api import create_video
from helper.speech_to_text_helper import speech_to_text, text_to_speech, has_words
from helper.speech_to_text_helper import speech_to_text, text_to_speech, has_words, has_10_words
from helper.token_counter import count_tokens
from helper.openai_interface import make_openai_call
import os
@@ -297,7 +297,7 @@ def grade_speaking_task_1():
download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
answer = speech_to_text(sound_file_name)
if has_words(answer):
if has_10_words(answer):
messages = get_grading_messages(QuestionType.SPEAKING_1, question, answer)
token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
@@ -306,7 +306,7 @@ def grade_speaking_task_1():
return response
else:
return {
"comment": "The audio recorded does not contain any english words.",
"comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0,
"task_response": {
"Fluency and Coherence": 0,
@@ -391,7 +391,7 @@ def grade_speaking_task_2():
download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
answer = speech_to_text(sound_file_name)
if has_words(answer):
if has_10_words(answer):
messages = get_grading_messages(QuestionType.SPEAKING_2, question, answer)
token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
@@ -400,7 +400,7 @@ def grade_speaking_task_2():
return response
else:
return {
"comment": "The audio recorded does not contain any english words.",
"comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0,
"task_response": {
"Fluency and Coherence": 0,

93
helper/api_messages.py
View File

@@ -101,75 +101,92 @@ def get_grading_messages(question_type: QuestionType, question: str, answer: str
return [
{
"role": "user",
"content": "You are a IELTS examiner.",
"content": "You are an IELTS examiner."
},
{
"role": "user",
"content": f"The question you have to grade is of type Speaking Task Part 1 and is the following: {question}",
"content": f"The question you need to grade is a Speaking Task Part 1 question, and it is as follows: {question}"
},
{
"role": "user",
"content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
"with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, "
"'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, "
"'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}",
"content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about "
"answer quality', 'overall': 7.0, 'task_response': {'Fluency and Coherence': 8.0, 'Lexical "
"Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}"
},
{
"role": "user",
"content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has "
"given examples of how they spend their weekends. However, there are some issues with grammar and "
"pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks "
"clearly and smoothly with only minor hesitations. They have also provided a well-organized response "
"that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary "
"related to weekend activities but there are some errors in word choice that affect the meaning of "
"their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple "
"and complex sentence structures but there are some errors in subject-verb agreement and preposition "
"use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some "
"issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, "
"'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0,"
" 'Pronunciation': 6.0}}",
"content": "Example output: {'comment': 'The candidate has provided a clear response to the question and "
"has given examples of how they spend their weekends. However, there are some issues with grammar "
"and pronunciation that affect the overall score. In terms of fluency and coherence, the candidate "
"speaks clearly and smoothly with only minor hesitations. They have also provided a well-organized "
"response that is easy to follow. Regarding lexical resource, the candidate has used a range of "
"vocabulary related to weekend activities but there are some errors in word choice that affect "
"the meaning of their sentences. In terms of grammatical range and accuracy, the candidate has "
"used a mix of simple and complex sentence structures but there are some errors in subject-verb "
"agreement and preposition use. Finally, regarding pronunciation, the candidate's speech is "
"generally clear but there are some issues with stress and intonation that make it difficult "
"to understand at times.', 'overall': 6.5, 'task_response': {'Fluency and Coherence': 7.0, "
"'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}"
},
{
"role": "user",
"content": f"Evaluate this answer according to ielts grading system: {answer}",
"content": "Please assign a grade of 0 if the answer provided does not address the question."
},
{
"role": "user",
"content": f"Assess this answer according to the IELTS grading system: {answer}"
},
{
"role": "user",
"content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, "
"and Pronunciation when grading the response."
}
]
elif QuestionType.SPEAKING_2 == question_type:
return [
{
"role": "user",
"content": "You are a IELTS examiner.",
"content": "You are an IELTS examiner."
},
{
"role": "user",
"content": f"The question you have to grade is of type Speaking Task Part 2 and is the following: {question}",
"content": f"The question you need to grade is a Speaking Task Part 2 question, and it is as follows: {question}"
},
{
"role": "user",
"content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
"with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, "
"'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, "
"'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}",
"content": "Please provide your assessment using the following JSON format: {\"comment\": \"Comment about "
"answer quality\", \"overall\": 7.0, \"task_response\": {\"Fluency and Coherence\": 8.0, \"Lexical "
"Resource\": 6.5, \"Grammatical Range and Accuracy\": 7.5, \"Pronunciation\": 6.0}}"
},
{
"role": "user",
"content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has "
"given examples of how they spend their weekends. However, there are some issues with grammar and "
"pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks "
"clearly and smoothly with only minor hesitations. They have also provided a well-organized response "
"that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary "
"related to weekend activities but there are some errors in word choice that affect the meaning of "
"their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple "
"and complex sentence structures but there are some errors in subject-verb agreement and preposition "
"use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some "
"issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, "
"'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0,"
" 'Pronunciation': 6.0}}",
"content": "Example output: {\"comment\": \"The candidate has provided a clear response to the question "
"and has given examples of how they spend their weekends. However, there are some issues with "
"grammar and pronunciation that affect the overall score. In terms of fluency and coherence, "
"the candidate speaks clearly and smoothly with only minor hesitations. They have also provided "
"a well-organized response that is easy to follow. Regarding lexical resource, the candidate "
"has used a range of vocabulary related to weekend activities but there are some errors in "
"word choice that affect the meaning of their sentences. In terms of grammatical range and "
"accuracy, the candidate has used a mix of simple and complex sentence structures but there "
"are some errors in subject-verb agreement and preposition use. Finally, regarding pronunciation, "
"the candidate's speech is generally clear but there are some issues with stress and intonation "
"that make it difficult to understand at times.\", \"overall\": 6.5, \"task_response\": {\"Fluency "
"and Coherence\": 7.0, \"Lexical Resource\": 6.5, \"Grammatical Range and Accuracy\": 7.0, "
"\"Pronunciation\": 6.0}}"
},
{
"role": "user",
"content": f"Evaluate this answer according to ielts grading system: {answer}",
"content": "Please assign a grade of 0 if the answer provided does not address the question."
},
{
"role": "user",
"content": f"Assess this answer according to the IELTS grading system: {answer}"
},
{
"role": "user",
"content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, "
"and Pronunciation when grading the response."
}
]
else:
raise Exception("Question type not implemented: " + question_type.value)

6
helper/speech_to_text_helper.py
View File

@@ -22,3 +22,9 @@ def has_words(text: str):
english_words = set(words.words())
words_in_input = text.split()
return any(word.lower() in english_words for word in words_in_input)
def has_10_words(text: str):
english_words = set(words.words())
words_in_input = text.split()
english_word_count = sum(1 for word in words_in_input if word.lower() in english_words)
return english_word_count >= 10