devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve speaking grading.

Browse Source
This commit is contained in:
Cristiano Ferreira
2023-09-15 00:01:52 +01:00
parent 8f9b65281e
commit f77fafa864
3 changed files with 67 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app.py
View File

@@ -8,7 +8,7 @@ from helper.file_helper import delete_files_older_than_one_day
from helper.firebase_helper import download_firebase_file, upload_file_firebase, upload_file_firebase_get_url, \ from helper.firebase_helper import download_firebase_file, upload_file_firebase, upload_file_firebase_get_url, \
save_to_db save_to_db
from helper.heygen_api import create_video from helper.heygen_api import create_video
from helper.speech_to_text_helper import speech_to_text, text_to_speech, has_words from helper.speech_to_text_helper import speech_to_text, text_to_speech, has_words, has_10_words
from helper.token_counter import count_tokens from helper.token_counter import count_tokens
from helper.openai_interface import make_openai_call from helper.openai_interface import make_openai_call
import os import os
@@ -297,7 +297,7 @@ def grade_speaking_task_1():
download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name) download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
answer = speech_to_text(sound_file_name) answer = speech_to_text(sound_file_name)
if has_words(answer): if has_10_words(answer):
messages = get_grading_messages(QuestionType.SPEAKING_1, question, answer) messages = get_grading_messages(QuestionType.SPEAKING_1, question, answer)
token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
@@ -306,7 +306,7 @@ def grade_speaking_task_1():
return response return response
else: else:
return { return {
"comment": "The audio recorded does not contain any english words.", "comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0, "overall": 0,
"task_response": { "task_response": {
"Fluency and Coherence": 0, "Fluency and Coherence": 0,
@@ -391,7 +391,7 @@ def grade_speaking_task_2():
download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name) download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
answer = speech_to_text(sound_file_name) answer = speech_to_text(sound_file_name)
if has_words(answer): if has_10_words(answer):
messages = get_grading_messages(QuestionType.SPEAKING_2, question, answer) messages = get_grading_messages(QuestionType.SPEAKING_2, question, answer)
token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
@@ -400,7 +400,7 @@ def grade_speaking_task_2():
return response return response
else: else:
return { return {
"comment": "The audio recorded does not contain any english words.", "comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0, "overall": 0,
"task_response": { "task_response": {
"Fluency and Coherence": 0, "Fluency and Coherence": 0,

93
helper/api_messages.py
View File

@@ -101,75 +101,92 @@ def get_grading_messages(question_type: QuestionType, question: str, answer: str
return [ return [
{ {
"role": "user", "role": "user",
"content": "You are a IELTS examiner.", "content": "You are an IELTS examiner."
}, },
{ {
"role": "user", "role": "user",
"content": f"The question you have to grade is of type Speaking Task Part 1 and is the following: {question}", "content": f"The question you need to grade is a Speaking Task Part 1 question, and it is as follows: {question}"
}, },
{ {
"role": "user", "role": "user",
"content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, " "content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about "
"with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, " "answer quality', 'overall': 7.0, 'task_response': {'Fluency and Coherence': 8.0, 'Lexical "
"'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, " "Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}"
"'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}",
}, },
{ {
"role": "user", "role": "user",
"content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has " "content": "Example output: {'comment': 'The candidate has provided a clear response to the question and "
"given examples of how they spend their weekends. However, there are some issues with grammar and " "has given examples of how they spend their weekends. However, there are some issues with grammar "
"pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks " "and pronunciation that affect the overall score. In terms of fluency and coherence, the candidate "
"clearly and smoothly with only minor hesitations. They have also provided a well-organized response " "speaks clearly and smoothly with only minor hesitations. They have also provided a well-organized "
"that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary " "response that is easy to follow. Regarding lexical resource, the candidate has used a range of "
"related to weekend activities but there are some errors in word choice that affect the meaning of " "vocabulary related to weekend activities but there are some errors in word choice that affect "
"their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple " "the meaning of their sentences. In terms of grammatical range and accuracy, the candidate has "
"and complex sentence structures but there are some errors in subject-verb agreement and preposition " "used a mix of simple and complex sentence structures but there are some errors in subject-verb "
"use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some " "agreement and preposition use. Finally, regarding pronunciation, the candidate's speech is "
"issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, " "generally clear but there are some issues with stress and intonation that make it difficult "
"'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0," "to understand at times.', 'overall': 6.5, 'task_response': {'Fluency and Coherence': 7.0, "
" 'Pronunciation': 6.0}}", "'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}"
}, },
{ {
"role": "user", "role": "user",
"content": f"Evaluate this answer according to ielts grading system: {answer}", "content": "Please assign a grade of 0 if the answer provided does not address the question."
}, },
{
"role": "user",
"content": f"Assess this answer according to the IELTS grading system: {answer}"
},
{
"role": "user",
"content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, "
"and Pronunciation when grading the response."
}
] ]
elif QuestionType.SPEAKING_2 == question_type: elif QuestionType.SPEAKING_2 == question_type:
return [ return [
{ {
"role": "user", "role": "user",
"content": "You are a IELTS examiner.", "content": "You are an IELTS examiner."
}, },
{ {
"role": "user", "role": "user",
"content": f"The question you have to grade is of type Speaking Task Part 2 and is the following: {question}", "content": f"The question you need to grade is a Speaking Task Part 2 question, and it is as follows: {question}"
}, },
{ {
"role": "user", "role": "user",
"content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, " "content": "Please provide your assessment using the following JSON format: {\"comment\": \"Comment about "
"with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, " "answer quality\", \"overall\": 7.0, \"task_response\": {\"Fluency and Coherence\": 8.0, \"Lexical "
"'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, " "Resource\": 6.5, \"Grammatical Range and Accuracy\": 7.5, \"Pronunciation\": 6.0}}"
"'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}",
}, },
{ {
"role": "user", "role": "user",
"content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has " "content": "Example output: {\"comment\": \"The candidate has provided a clear response to the question "
"given examples of how they spend their weekends. However, there are some issues with grammar and " "and has given examples of how they spend their weekends. However, there are some issues with "
"pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks " "grammar and pronunciation that affect the overall score. In terms of fluency and coherence, "
"clearly and smoothly with only minor hesitations. They have also provided a well-organized response " "the candidate speaks clearly and smoothly with only minor hesitations. They have also provided "
"that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary " "a well-organized response that is easy to follow. Regarding lexical resource, the candidate "
"related to weekend activities but there are some errors in word choice that affect the meaning of " "has used a range of vocabulary related to weekend activities but there are some errors in "
"their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple " "word choice that affect the meaning of their sentences. In terms of grammatical range and "
"and complex sentence structures but there are some errors in subject-verb agreement and preposition " "accuracy, the candidate has used a mix of simple and complex sentence structures but there "
"use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some " "are some errors in subject-verb agreement and preposition use. Finally, regarding pronunciation, "
"issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, " "the candidate's speech is generally clear but there are some issues with stress and intonation "
"'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0," "that make it difficult to understand at times.\", \"overall\": 6.5, \"task_response\": {\"Fluency "
" 'Pronunciation': 6.0}}", "and Coherence\": 7.0, \"Lexical Resource\": 6.5, \"Grammatical Range and Accuracy\": 7.0, "
"\"Pronunciation\": 6.0}}"
}, },
{ {
"role": "user", "role": "user",
"content": f"Evaluate this answer according to ielts grading system: {answer}", "content": "Please assign a grade of 0 if the answer provided does not address the question."
}, },
{
"role": "user",
"content": f"Assess this answer according to the IELTS grading system: {answer}"
},
{
"role": "user",
"content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, "
"and Pronunciation when grading the response."
}
] ]
else: else:
raise Exception("Question type not implemented: " + question_type.value) raise Exception("Question type not implemented: " + question_type.value)

6
helper/speech_to_text_helper.py
View File

@@ -22,3 +22,9 @@ def has_words(text: str):
english_words = set(words.words()) english_words = set(words.words())
words_in_input = text.split() words_in_input = text.split()
return any(word.lower() in english_words for word in words_in_input) return any(word.lower() in english_words for word in words_in_input)
def has_10_words(text: str):
english_words = set(words.words())
words_in_input = text.split()
english_word_count = sum(1 for word in words_in_input if word.lower() in english_words)
return english_word_count >= 10