From 8d9cd2949c982ca915e66b9d413bd76a912ebf4b Mon Sep 17 00:00:00 2001 From: Cristiano Ferreira Date: Sat, 16 Sep 2023 11:44:08 +0100 Subject: [PATCH] Add speaking task 3 grading endpoint. --- app.py | 37 +++++++++++++- helper/api_messages.py | 69 +++++++++++++++++++++------ postman/ielts.postman_collection.json | 10 ++-- 3 files changed, 96 insertions(+), 20 deletions(-) diff --git a/app.py b/app.py index 82bd1cb..5fec16a 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,8 @@ from flask_jwt_extended import JWTManager, jwt_required from functools import reduce import firebase_admin from firebase_admin import credentials, firestore -from helper.api_messages import QuestionType, get_grading_messages, get_question_gen_messages, get_question_tips +from helper.api_messages import QuestionType, get_grading_messages, get_question_gen_messages, get_question_tips, \ + get_speaking_grading_messages from helper.file_helper import delete_files_older_than_one_day from helper.firebase_helper import download_firebase_file, upload_file_firebase, upload_file_firebase_get_url, \ save_to_db @@ -473,6 +474,40 @@ def save_speaking_task_2_question(): except Exception as e: return str(e) +@app.route('/speaking_task_3', methods=['POST']) +@jwt_required() +def grade_speaking_task_3(): + delete_files_older_than_one_day(AUDIO_FILES_PATH) + try: + data = request.get_json() + answers = data.get('answers') + + for item in answers: + sound_file_name = AUDIO_FILES_PATH + str(uuid.uuid4()) + download_firebase_file(FIREBASE_BUCKET, item["answer"], sound_file_name) + answer_text = speech_to_text(sound_file_name) + item["answer_text"] = answer_text + os.remove(sound_file_name) + if not has_10_words(answer_text): + return { + "comment": "The audio recorded does not contain enough english words to be graded.", + "overall": 0, + "task_response": { + "Fluency and Coherence": 0, + "Lexical Resource": 0, + "Grammatical Range and Accuracy": 0, + "Pronunciation": 0 + } + } + + messages = get_speaking_grading_messages(answers) + token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], + map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) + response = make_openai_call(GPT_3_5_TURBO, messages, token_count, GRADING_FIELDS, GRADING_TEMPERATURE) + return response + except Exception as e: + return str(e), 400 + @app.route('/save_speaking_task_3', methods=['POST']) @jwt_required() def save_speaking_task_3_question(): diff --git a/helper/api_messages.py b/helper/api_messages.py index 5f31b24..881fc74 100644 --- a/helper/api_messages.py +++ b/helper/api_messages.py @@ -1,5 +1,7 @@ from enum import Enum +from typing import List + class QuestionType(Enum): LISTENING_SECTION_1 = "Listening Section 1" @@ -109,23 +111,14 @@ def get_grading_messages(question_type: QuestionType, question: str, answer: str }, { "role": "user", - "content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about " - "answer quality', 'overall': 7.0, 'task_response': {'Fluency and Coherence': 8.0, 'Lexical " - "Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}" + "content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about answer " + "quality will go here', 'overall': 7.0, 'task_response': {'Fluency and " + "Coherence': 8.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}" }, { "role": "user", - "content": "Example output: {'comment': 'The candidate has provided a clear response to the question and " - "has given examples of how they spend their weekends. However, there are some issues with grammar " - "and pronunciation that affect the overall score. In terms of fluency and coherence, the candidate " - "speaks clearly and smoothly with only minor hesitations. They have also provided a well-organized " - "response that is easy to follow. Regarding lexical resource, the candidate has used a range of " - "vocabulary related to weekend activities but there are some errors in word choice that affect " - "the meaning of their sentences. In terms of grammatical range and accuracy, the candidate has " - "used a mix of simple and complex sentence structures but there are some errors in subject-verb " - "agreement and preposition use. Finally, regarding pronunciation, the candidate's speech is " - "generally clear but there are some issues with stress and intonation that make it difficult " - "to understand at times.', 'overall': 6.5, 'task_response': {'Fluency and Coherence': 7.0, " + "content": "Example output: {'comment': 'Comment about answer quality will go here', 'overall': 6.5, " + "'task_response': {'Fluency and Coherence': 7.0, " "'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}" }, { @@ -192,6 +185,54 @@ def get_grading_messages(question_type: QuestionType, question: str, answer: str raise Exception("Question type not implemented: " + question_type.value) +def get_speaking_grading_messages(answers: List): + messages = [ + { + "role": "user", + "content": "You are an IELTS examiner." + }, + { + "role": "user", + "content": "The exercise you need to grade is a Speaking Task, and it is has the following questions and answers:" + } + ] + for item in answers: + question = item["question"] + answer = item["answer_text"] + messages.append({ + "role": "user", + "content": f"Question: {question}; Answer: {answer}" + }) + messages.extend([ + { + "role": "user", + "content": f"Assess this answer according to the IELTS grading system." + }, + { + "role": "user", + "content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about answer " + "quality will go here', 'overall': 7.0, 'task_response': {'Fluency and " + "Coherence': 8.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}" + }, + { + "role": "user", + "content": "Example output: {'comment': 'Comment about answer quality will go here', 'overall': 6.5, " + "'task_response': {'Fluency and Coherence': 7.0, " + "'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}" + }, + { + "role": "user", + "content": "Please assign a grade of 0 if the answer provided does not address the question." + }, + { + "role": "user", + "content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, " + "and Pronunciation when grading the response." + } + ]) + return messages + + def get_question_gen_messages(question_type: QuestionType): if QuestionType.LISTENING_SECTION_1 == question_type: return [ diff --git a/postman/ielts.postman_collection.json b/postman/ielts.postman_collection.json index 6824066..1b59f89 100644 --- a/postman/ielts.postman_collection.json +++ b/postman/ielts.postman_collection.json @@ -1,9 +1,9 @@ { "info": { - "_postman_id": "c3a09737-c624-4b32-9e9a-af8ee8084764", + "_postman_id": "1b901158-4228-426a-9c96-8cedc4df8470", "name": "ielts", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "29491168" + "_exporter_id": "26107457" }, "item": [ { @@ -190,7 +190,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\r\n \"question\": \"How do you usually spend your weekends? Why?\",\r\n \"answer\": \"speaking_recordings/weekends.m4a\"\r\n}", + "raw": " {\r\n \"answers\": [\r\n {\r\n \"question\": \"How do you think technology has affected the way people communicate with each other in today's society?\",\r\n \"answer\": \"speaking_recordings/weekends.m4a\"\r\n },\r\n {\r\n \"question\": \"In what ways has the use of smartphones and social media platforms changed the dynamics of personal relationships?\",\r\n \"answer\": \"speaking_recordings/weekends.m4a\"\r\n },\r\n {\r\n \"question\": \"Some argue that technology has made communication more convenient, while others worry that it has led to a decline in face-to-face interactions. What's your perspective on this matter, and how do you think it might impact future generations?\",\r\n \"answer\": \"speaking_recordings/weekends.m4a\"\r\n }\r\n ]\r\n}", "options": { "raw": { "language": "json" @@ -198,7 +198,7 @@ } }, "url": { - "raw": "http://127.0.0.1:5000/speaking_task", + "raw": "http://127.0.0.1:5000/speaking_task_3", "protocol": "http", "host": [ "127", @@ -208,7 +208,7 @@ ], "port": "5000", "path": [ - "speaking_task" + "speaking_task_3" ] } },