devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add speaking task 3 grading endpoint.

Browse Source
This commit is contained in:
Cristiano Ferreira
2023-09-16 11:44:08 +01:00
parent f77fafa864
commit 8d9cd2949c
3 changed files with 96 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
app.py
View File

@@ -3,7 +3,8 @@ from flask_jwt_extended import JWTManager, jwt_required
from functools import reduce
import firebase_admin
from firebase_admin import credentials, firestore
from helper.api_messages import QuestionType, get_grading_messages, get_question_gen_messages, get_question_tips
from helper.api_messages import QuestionType, get_grading_messages, get_question_gen_messages, get_question_tips, \
get_speaking_grading_messages
from helper.file_helper import delete_files_older_than_one_day
from helper.firebase_helper import download_firebase_file, upload_file_firebase, upload_file_firebase_get_url, \
save_to_db
@@ -473,6 +474,40 @@ def save_speaking_task_2_question():
except Exception as e:
return str(e)
@app.route('/speaking_task_3', methods=['POST'])
@jwt_required()
def grade_speaking_task_3():
delete_files_older_than_one_day(AUDIO_FILES_PATH)
try:
data = request.get_json()
answers = data.get('answers')
for item in answers:
sound_file_name = AUDIO_FILES_PATH + str(uuid.uuid4())
download_firebase_file(FIREBASE_BUCKET, item["answer"], sound_file_name)
answer_text = speech_to_text(sound_file_name)
item["answer_text"] = answer_text
os.remove(sound_file_name)
if not has_10_words(answer_text):
return {
"comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0,
"task_response": {
"Fluency and Coherence": 0,
"Lexical Resource": 0,
"Grammatical Range and Accuracy": 0,
"Pronunciation": 0
}
}
messages = get_speaking_grading_messages(answers)
token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
response = make_openai_call(GPT_3_5_TURBO, messages, token_count, GRADING_FIELDS, GRADING_TEMPERATURE)
return response
except Exception as e:
return str(e), 400
@app.route('/save_speaking_task_3', methods=['POST'])
@jwt_required()
def save_speaking_task_3_question():

69
helper/api_messages.py
View File

@@ -1,5 +1,7 @@
from enum import Enum
from typing import List
class QuestionType(Enum):
LISTENING_SECTION_1 = "Listening Section 1"
@@ -109,23 +111,14 @@ def get_grading_messages(question_type: QuestionType, question: str, answer: str
},
{
"role": "user",
"content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about "
"answer quality', 'overall': 7.0, 'task_response': {'Fluency and Coherence': 8.0, 'Lexical "
"Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}"
"content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about answer "
"quality will go here', 'overall': 7.0, 'task_response': {'Fluency and "
"Coherence': 8.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}"
},
{
"role": "user",
"content": "Example output: {'comment': 'The candidate has provided a clear response to the question and "
"has given examples of how they spend their weekends. However, there are some issues with grammar "
"and pronunciation that affect the overall score. In terms of fluency and coherence, the candidate "
"speaks clearly and smoothly with only minor hesitations. They have also provided a well-organized "
"response that is easy to follow. Regarding lexical resource, the candidate has used a range of "
"vocabulary related to weekend activities but there are some errors in word choice that affect "
"the meaning of their sentences. In terms of grammatical range and accuracy, the candidate has "
"used a mix of simple and complex sentence structures but there are some errors in subject-verb "
"agreement and preposition use. Finally, regarding pronunciation, the candidate's speech is "
"generally clear but there are some issues with stress and intonation that make it difficult "
"to understand at times.', 'overall': 6.5, 'task_response': {'Fluency and Coherence': 7.0, "
"content": "Example output: {'comment': 'Comment about answer quality will go here', 'overall': 6.5, "
"'task_response': {'Fluency and Coherence': 7.0, "
"'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}"
},
{
@@ -192,6 +185,54 @@ def get_grading_messages(question_type: QuestionType, question: str, answer: str
raise Exception("Question type not implemented: " + question_type.value)
def get_speaking_grading_messages(answers: List):
messages = [
{
"role": "user",
"content": "You are an IELTS examiner."
},
{
"role": "user",
"content": "The exercise you need to grade is a Speaking Task, and it is has the following questions and answers:"
}
]
for item in answers:
question = item["question"]
answer = item["answer_text"]
messages.append({
"role": "user",
"content": f"Question: {question}; Answer: {answer}"
})
messages.extend([
{
"role": "user",
"content": f"Assess this answer according to the IELTS grading system."
},
{
"role": "user",
"content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about answer "
"quality will go here', 'overall': 7.0, 'task_response': {'Fluency and "
"Coherence': 8.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}"
},
{
"role": "user",
"content": "Example output: {'comment': 'Comment about answer quality will go here', 'overall': 6.5, "
"'task_response': {'Fluency and Coherence': 7.0, "
"'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}"
},
{
"role": "user",
"content": "Please assign a grade of 0 if the answer provided does not address the question."
},
{
"role": "user",
"content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, "
"and Pronunciation when grading the response."
}
])
return messages
def get_question_gen_messages(question_type: QuestionType):
if QuestionType.LISTENING_SECTION_1 == question_type:
return [

10
postman/ielts.postman_collection.json
View File

@@ -1,9 +1,9 @@
{
"info": {
"_postman_id": "c3a09737-c624-4b32-9e9a-af8ee8084764",
"_postman_id": "1b901158-4228-426a-9c96-8cedc4df8470",
"name": "ielts",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
"_exporter_id": "29491168"
"_exporter_id": "26107457"
},
"item": [
{
@@ -190,7 +190,7 @@
"header": [],
"body": {
"mode": "raw",
"raw": "{\r\n \"question\": \"How do you usually spend your weekends? Why?\",\r\n \"answer\": \"speaking_recordings/weekends.m4a\"\r\n}",
"raw": " {\r\n \"answers\": [\r\n {\r\n \"question\": \"How do you think technology has affected the way people communicate with each other in today's society?\",\r\n \"answer\": \"speaking_recordings/weekends.m4a\"\r\n },\r\n {\r\n \"question\": \"In what ways has the use of smartphones and social media platforms changed the dynamics of personal relationships?\",\r\n \"answer\": \"speaking_recordings/weekends.m4a\"\r\n },\r\n {\r\n \"question\": \"Some argue that technology has made communication more convenient, while others worry that it has led to a decline in face-to-face interactions. What's your perspective on this matter, and how do you think it might impact future generations?\",\r\n \"answer\": \"speaking_recordings/weekends.m4a\"\r\n }\r\n ]\r\n}",
"options": {
"raw": {
"language": "json"
@@ -198,7 +198,7 @@
}
},
"url": {
"raw": "http://127.0.0.1:5000/speaking_task",
"raw": "http://127.0.0.1:5000/speaking_task_3",
"protocol": "http",
"host": [
"127",
@@ -208,7 +208,7 @@
],
"port": "5000",
"path": [
"speaking_task"
"speaking_task_3"
]
}
},