diff --git a/app.py b/app.py index 5b60928..82bd1cb 100644 --- a/app.py +++ b/app.py @@ -8,7 +8,7 @@ from helper.file_helper import delete_files_older_than_one_day from helper.firebase_helper import download_firebase_file, upload_file_firebase, upload_file_firebase_get_url, \ save_to_db from helper.heygen_api import create_video -from helper.speech_to_text_helper import speech_to_text, text_to_speech, has_words +from helper.speech_to_text_helper import speech_to_text, text_to_speech, has_words, has_10_words from helper.token_counter import count_tokens from helper.openai_interface import make_openai_call import os @@ -297,7 +297,7 @@ def grade_speaking_task_1(): download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name) answer = speech_to_text(sound_file_name) - if has_words(answer): + if has_10_words(answer): messages = get_grading_messages(QuestionType.SPEAKING_1, question, answer) token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) @@ -306,7 +306,7 @@ def grade_speaking_task_1(): return response else: return { - "comment": "The audio recorded does not contain any english words.", + "comment": "The audio recorded does not contain enough english words to be graded.", "overall": 0, "task_response": { "Fluency and Coherence": 0, @@ -391,7 +391,7 @@ def grade_speaking_task_2(): download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name) answer = speech_to_text(sound_file_name) - if has_words(answer): + if has_10_words(answer): messages = get_grading_messages(QuestionType.SPEAKING_2, question, answer) token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) @@ -400,7 +400,7 @@ def grade_speaking_task_2(): return response else: return { - "comment": "The audio recorded does not contain any english words.", + "comment": "The audio recorded does not contain enough english words to be graded.", "overall": 0, "task_response": { "Fluency and Coherence": 0, diff --git a/helper/api_messages.py b/helper/api_messages.py index 00a5ecc..5f31b24 100644 --- a/helper/api_messages.py +++ b/helper/api_messages.py @@ -101,75 +101,92 @@ def get_grading_messages(question_type: QuestionType, question: str, answer: str return [ { "role": "user", - "content": "You are a IELTS examiner.", + "content": "You are an IELTS examiner." }, { "role": "user", - "content": f"The question you have to grade is of type Speaking Task Part 1 and is the following: {question}", + "content": f"The question you need to grade is a Speaking Task Part 1 question, and it is as follows: {question}" }, { "role": "user", - "content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, " - "with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, " - "'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, " - "'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}", + "content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about " + "answer quality', 'overall': 7.0, 'task_response': {'Fluency and Coherence': 8.0, 'Lexical " + "Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}" }, { "role": "user", - "content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has " - "given examples of how they spend their weekends. However, there are some issues with grammar and " - "pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks " - "clearly and smoothly with only minor hesitations. They have also provided a well-organized response " - "that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary " - "related to weekend activities but there are some errors in word choice that affect the meaning of " - "their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple " - "and complex sentence structures but there are some errors in subject-verb agreement and preposition " - "use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some " - "issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, " - "'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0," - " 'Pronunciation': 6.0}}", + "content": "Example output: {'comment': 'The candidate has provided a clear response to the question and " + "has given examples of how they spend their weekends. However, there are some issues with grammar " + "and pronunciation that affect the overall score. In terms of fluency and coherence, the candidate " + "speaks clearly and smoothly with only minor hesitations. They have also provided a well-organized " + "response that is easy to follow. Regarding lexical resource, the candidate has used a range of " + "vocabulary related to weekend activities but there are some errors in word choice that affect " + "the meaning of their sentences. In terms of grammatical range and accuracy, the candidate has " + "used a mix of simple and complex sentence structures but there are some errors in subject-verb " + "agreement and preposition use. Finally, regarding pronunciation, the candidate's speech is " + "generally clear but there are some issues with stress and intonation that make it difficult " + "to understand at times.', 'overall': 6.5, 'task_response': {'Fluency and Coherence': 7.0, " + "'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}" }, { "role": "user", - "content": f"Evaluate this answer according to ielts grading system: {answer}", + "content": "Please assign a grade of 0 if the answer provided does not address the question." }, + { + "role": "user", + "content": f"Assess this answer according to the IELTS grading system: {answer}" + }, + { + "role": "user", + "content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, " + "and Pronunciation when grading the response." + } ] elif QuestionType.SPEAKING_2 == question_type: return [ { "role": "user", - "content": "You are a IELTS examiner.", + "content": "You are an IELTS examiner." }, { "role": "user", - "content": f"The question you have to grade is of type Speaking Task Part 2 and is the following: {question}", + "content": f"The question you need to grade is a Speaking Task Part 2 question, and it is as follows: {question}" }, { "role": "user", - "content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, " - "with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, " - "'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, " - "'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}", + "content": "Please provide your assessment using the following JSON format: {\"comment\": \"Comment about " + "answer quality\", \"overall\": 7.0, \"task_response\": {\"Fluency and Coherence\": 8.0, \"Lexical " + "Resource\": 6.5, \"Grammatical Range and Accuracy\": 7.5, \"Pronunciation\": 6.0}}" }, { "role": "user", - "content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has " - "given examples of how they spend their weekends. However, there are some issues with grammar and " - "pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks " - "clearly and smoothly with only minor hesitations. They have also provided a well-organized response " - "that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary " - "related to weekend activities but there are some errors in word choice that affect the meaning of " - "their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple " - "and complex sentence structures but there are some errors in subject-verb agreement and preposition " - "use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some " - "issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, " - "'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0," - " 'Pronunciation': 6.0}}", + "content": "Example output: {\"comment\": \"The candidate has provided a clear response to the question " + "and has given examples of how they spend their weekends. However, there are some issues with " + "grammar and pronunciation that affect the overall score. In terms of fluency and coherence, " + "the candidate speaks clearly and smoothly with only minor hesitations. They have also provided " + "a well-organized response that is easy to follow. Regarding lexical resource, the candidate " + "has used a range of vocabulary related to weekend activities but there are some errors in " + "word choice that affect the meaning of their sentences. In terms of grammatical range and " + "accuracy, the candidate has used a mix of simple and complex sentence structures but there " + "are some errors in subject-verb agreement and preposition use. Finally, regarding pronunciation, " + "the candidate's speech is generally clear but there are some issues with stress and intonation " + "that make it difficult to understand at times.\", \"overall\": 6.5, \"task_response\": {\"Fluency " + "and Coherence\": 7.0, \"Lexical Resource\": 6.5, \"Grammatical Range and Accuracy\": 7.0, " + "\"Pronunciation\": 6.0}}" }, { "role": "user", - "content": f"Evaluate this answer according to ielts grading system: {answer}", + "content": "Please assign a grade of 0 if the answer provided does not address the question." }, + { + "role": "user", + "content": f"Assess this answer according to the IELTS grading system: {answer}" + }, + { + "role": "user", + "content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, " + "and Pronunciation when grading the response." + } ] else: raise Exception("Question type not implemented: " + question_type.value) diff --git a/helper/speech_to_text_helper.py b/helper/speech_to_text_helper.py index 76669bb..6ae02fc 100644 --- a/helper/speech_to_text_helper.py +++ b/helper/speech_to_text_helper.py @@ -21,4 +21,10 @@ def text_to_speech(text: str, file_name: str): def has_words(text: str): english_words = set(words.words()) words_in_input = text.split() - return any(word.lower() in english_words for word in words_in_input) \ No newline at end of file + return any(word.lower() in english_words for word in words_in_input) + +def has_10_words(text: str): + english_words = set(words.words()) + words_in_input = text.split() + english_word_count = sum(1 for word in words_in_input if word.lower() in english_words) + return english_word_count >= 10 \ No newline at end of file