devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged in speaking-improvements (pull request #9)

Browse Source 
Speaking improvements
This commit is contained in:
Cristiano Ferreira
2024-06-17 13:06:15 +00:00
parent a4caecdb4f ee5f23b3d7
commit ad3a32ce45
3 changed files with 359 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

400
app.py
View File

@@ -419,48 +419,56 @@ def get_writing_task_2_general_question():
def grade_speaking_task_1():
request_id = uuid.uuid4()
delete_files_older_than_one_day(AUDIO_FILES_PATH)
sound_file_name = AUDIO_FILES_PATH + str(uuid.uuid4())
logging.info("POST - speaking_task_1 - Received request to grade speaking task 1. "
"Use this id to track the logs: " + str(request_id) + " - Request data: " + str(request.get_json()))
try:
data = request.get_json()
question = data.get('question')
answer_firebase_path = data.get('answer')
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Downloading file " + answer_firebase_path)
download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
answers = data.get('answers')
text_answers = []
perfect_answers = []
logging.info("POST - speaking_task_1 - " + str(
request_id) + " - Downloaded file " + answer_firebase_path + " to " + sound_file_name)
request_id) + " - Received " + str(len(answers)) + " total answers.")
answer = speech_to_text(sound_file_name)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Transcripted answer: " + answer)
for item in answers:
sound_file_name = AUDIO_FILES_PATH + str(uuid.uuid4())
if has_x_words(answer, 20):
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"comment": "comment about answer quality", "overall": 0.0, '
'"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, '
'"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}')
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Downloading file " + item["answer"])
download_firebase_file(FIREBASE_BUCKET, item["answer"], sound_file_name)
logging.info("POST - speaking_task_1 - " + str(
request_id) + " - Downloaded file " + item["answer"] + " to " + sound_file_name)
answer_text = speech_to_text(sound_file_name)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Transcripted answer: " + answer_text)
text_answers.append(answer_text)
item["answer"] = answer_text
os.remove(sound_file_name)
if not has_x_words(answer_text, 20):
logging.info("POST - speaking_task_1 - " + str(
request_id) + " - The answer had less words than threshold 20 to be graded. Answer: " + answer_text)
return {
"comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0,
"task_response": {
"Fluency and Coherence": {
"grade": 0.0,
"comment": ""
},
{
"role": "user",
"content": (
'Evaluate the given Speaking Part 1 response based on the IELTS grading system, ensuring a '
'strict assessment that penalizes errors. Deduct points for deviations from the task, and '
'assign a score of 0 if the response fails to address the question. Additionally, provide '
'detailed commentary highlighting both strengths and weaknesses in the response.'
'\n Question: "' + question + '" \n Answer: "' + answer + '"')
"Lexical Resource": {
"grade": 0.0,
"comment": ""
},
"Grammatical Range and Accuracy": {
"grade": 0.0,
"comment": ""
},
"Pronunciation": {
"grade": 0.0,
"comment": ""
}
}
}
]
token_count = count_total_tokens(messages)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Requesting grading of the answer.")
response = make_openai_call(GPT_3_5_TURBO, messages, token_count, ["comment"],
GRADING_TEMPERATURE)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Answer graded: " + str(response))
perfect_answer_messages = [
{
@@ -472,25 +480,101 @@ def grade_speaking_task_1():
"role": "user",
"content": (
'Provide a perfect answer according to ielts grading system to the following '
'Speaking Part 1 question: "' + question + '"')
'Speaking Part 1 question: "' + item["question"] + '"')
},
{
"role": "user",
"content": 'The answer must be 2 or 3 sentences long.'
}
]
token_count = count_total_tokens(perfect_answer_messages)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Requesting perfect answer.")
response['perfect_answer'] = make_openai_call(GPT_3_5_TURBO,
token_count = count_total_tokens(perfect_answer_messages)
logging.info("POST - speaking_task_1 - " + str(
request_id) + " - Requesting perfect answer for question: " + item["question"])
perfect_answers.append(make_openai_call(GPT_4_O,
perfect_answer_messages,
token_count,
["answer"],
GEN_QUESTION_TEMPERATURE)["answer"]
GEN_QUESTION_TEMPERATURE))
json_format = {
"comment": "comment about answers quality",
"overall": 0.0,
"task_response": {
"Fluency and Coherence": {
"grade": 0.0,
"comment": "comment about fluency and coherence"
},
"Lexical Resource": {
"grade": 0.0,
"comment": "comment about lexical resource"
},
"Grammatical Range and Accuracy": {
"grade": 0.0,
"comment": "comment about grammatical range and accuracy"
},
"Pronunciation": {
"grade": 0.0,
"comment": "comment about pronunciation on the transcribed answers"
}
}
}
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Formatting answers and questions for prompt.")
formatted_text = ""
for i, entry in enumerate(answers, start=1):
formatted_text += f"**Question {i}:**\n{entry['question']}\n\n"
formatted_text += f"**Answer {i}:**\n{entry['answer']}\n\n"
logging.info("POST - speaking_task_1 - " + str(
request_id) + " - Perfect answer: " + response['perfect_answer'])
request_id) + " - Formatted answers and questions for prompt: " + formatted_text)
response['transcript'] = answer
grade_message = (
'Evaluate the given Speaking Part 1 response based on the IELTS grading system, ensuring a '
'strict assessment that penalizes errors. Deduct points for deviations from the task, and '
'assign a score of 0 if the response fails to address the question. Additionally, provide '
'detailed commentary highlighting both strengths and weaknesses in the response.'
"\n\n The questions and answers are: \n\n'" + formatted_text)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Requesting fixed text.")
response['fixed_text'] = get_speaking_corrections(answer)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Fixed text: " + response['fixed_text'])
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: ' + str(json_format))
},
{
"role": "user",
"content": grade_message
},
{
"role": "user",
"content": 'Address the student as "you". If the answers are not 2 or 3 sentences long, warn the '
'student that they should be.'
},
{
"role": "user",
"content": 'For pronunciations act as if you heard the answers and they were transcripted as you heard them.'
},
{
"role": "user",
"content": 'The comments must be long, detailed, justify the grading and suggest improvements.'
}
]
token_count = count_total_tokens(messages)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Requesting grading of the answer.")
response = make_openai_call(GPT_4_O, messages, token_count, ["comment"],
GRADING_TEMPERATURE)
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Answers graded: " + str(response))
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Adding perfect answers to response.")
for i, answer in enumerate(perfect_answers, start=1):
response['perfect_answer_' + str(i)] = answer
logging.info("POST - speaking_task_1 - " + str(
request_id) + " - Adding transcript and fixed texts to response.")
for i, answer in enumerate(text_answers, start=1):
response['transcript_' + str(i)] = answer
response['fixed_text_' + str(i)] = get_speaking_corrections(answer)
if response["overall"] == "0.0" or response["overall"] == 0.0:
response["overall"] = round((response["task_response"]["Fluency and Coherence"] +
@@ -500,21 +584,7 @@ def grade_speaking_task_1():
logging.info("POST - speaking_task_1 - " + str(request_id) + " - Final response: " + str(response))
return response
else:
logging.info("POST - speaking_task_1 - " + str(
request_id) + " - The answer had less words than threshold 20 to be graded. Answer: " + answer)
return {
"comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0,
"task_response": {
"Fluency and Coherence": 0,
"Lexical Resource": 0,
"Grammatical Range and Accuracy": 0,
"Pronunciation": 0
}
}
except Exception as e:
os.remove(sound_file_name)
return str(e), 400
@@ -522,22 +592,35 @@ def grade_speaking_task_1():
@jwt_required()
def get_speaking_task_1_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics))
first_topic = request.args.get("first_topic", default=random.choice(mti_topics))
second_topic = request.args.get("second_topic", default=random.choice(mti_topics))
json_format = {
"first_topic": "topic 1",
"second_topic": "topic 2",
"questions": [
"Introductory question, should start with a greeting and introduce a question about the first topic.",
"Follow up question about the first topic",
"Follow up question about the first topic",
"Question about second topic",
"Follow up question about the second topic",
]
}
try:
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"topic": "topic", "question": "question"}')
'You are a helpful assistant designed to output JSON on this format: ' + str(json_format))
},
{
"role": "user",
"content": (
'Craft a thought-provoking question of ' + difficulty + ' difficulty for IELTS Speaking Part 1 '
'Craft 5 thought-provoking questions of ' + difficulty + ' difficulty for IELTS Speaking Part 1 '
'that encourages candidates to delve deeply into '
'personal experiences, preferences, or insights on the topic '
'of "' + topic + '". Instruct the candidate '
'of "' + first_topic + '" and the topic of "' + second_topic + '". Instruct the candidate '
'to offer not only detailed '
'descriptions but also provide '
'nuanced explanations, examples, '
@@ -545,14 +628,17 @@ def get_speaking_task_1_question():
'Make sure that the generated question '
'does not contain forbidden subjects in '
'muslim countries.')
},
{
"role": "user",
"content": 'The questions should lead to the usage of 4 verb tenses (present perfect, present, past and future).'
}
]
token_count = count_total_tokens(messages)
response = make_openai_call(GPT_4_O, messages, token_count, ["topic"],
response = make_openai_call(GPT_4_O, messages, token_count, ["first_topic"],
GEN_QUESTION_TEMPERATURE)
response["type"] = 1
response["difficulty"] = difficulty
response["topic"] = topic
return response
except Exception as e:
return str(e)
@@ -579,15 +665,35 @@ def grade_speaking_task_2():
answer = speech_to_text(sound_file_name)
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Transcripted answer: " + answer)
json_format = {
"comment": "extensive comment about answer quality",
"overall": 0.0,
"task_response": {
"Fluency and Coherence": {
"grade": 0.0,
"comment": "extensive comment about fluency and coherence, use examples to justify the grade awarded."
},
"Lexical Resource": {
"grade": 0.0,
"comment": "extensive comment about lexical resource, use examples to justify the grade awarded."
},
"Grammatical Range and Accuracy": {
"grade": 0.0,
"comment": "extensive comment about grammatical range and accuracy, use examples to justify the grade awarded."
},
"Pronunciation": {
"grade": 0.0,
"comment": "extensive comment about pronunciation on the transcribed answer, use examples to justify the grade awarded."
}
}
}
if has_x_words(answer, 20):
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"comment": "comment about answer quality", "overall": 0.0, '
'"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, '
'"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}')
'You are a helpful assistant designed to output JSON on this format: ' + str(json_format))
},
{
"role": "user",
@@ -597,6 +703,10 @@ def grade_speaking_task_2():
'assign a score of 0 if the response fails to address the question. Additionally, provide '
'detailed commentary highlighting both strengths and weaknesses in the response.'
'\n Question: "' + question + '" \n Answer: "' + answer + '"')
},
{
"role": "user",
"content": 'Address the student as "you"'
}
]
token_count = count_total_tokens(messages)
@@ -651,10 +761,22 @@ def grade_speaking_task_2():
"comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0,
"task_response": {
"Fluency and Coherence": 0,
"Lexical Resource": 0,
"Grammatical Range and Accuracy": 0,
"Pronunciation": 0
"Fluency and Coherence": {
"grade": 0.0,
"comment": ""
},
"Lexical Resource": {
"grade": 0.0,
"comment": ""
},
"Grammatical Range and Accuracy": {
"grade": 0.0,
"comment": ""
},
"Pronunciation": {
"grade": 0.0,
"comment": ""
}
}
}
except Exception as e:
@@ -667,21 +789,31 @@ def grade_speaking_task_2():
def get_speaking_task_2_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics))
json_format = {
"topic": "topic",
"question": "question",
"prompts": [
"prompt_1",
"prompt_2",
"prompt_3"
],
"suffix": "And explain why..."
}
try:
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"topic": "topic", "question": "question", "prompts": ["prompt_1", "prompt_2", "prompt_3"]}')
"content": 'You are a helpful assistant designed to output JSON on this format: ' + str(json_format)
},
{
"role": "user",
"content": (
'Create a question of ' + difficulty + ' difficulty for IELTS Speaking Part 2 '
'Create a question of medium difficulty for IELTS Speaking Part 2 '
'that encourages candidates to narrate a '
'personal experience or story related to the topic '
'of "' + topic + '". Include 3 prompts that '
'of "' + random.choice(mti_topics) + '". Include 3 prompts that '
'guide the candidate to describe '
'specific aspects of the experience, '
'such as details about the situation, '
@@ -689,6 +821,11 @@ def get_speaking_task_2_question():
'lasting impression. Make sure that the '
'generated question does not contain '
'forbidden subjects in muslim countries.')
},
{
"role": "user",
"content": 'The prompts must not be questions. Also include a suffix like the ones in the IELTS exams '
'that start with "And explain why".'
}
]
token_count = count_total_tokens(messages)
@@ -706,19 +843,30 @@ def get_speaking_task_2_question():
def get_speaking_task_3_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics))
json_format = {
"topic": "topic",
"questions": [
"Introductory question, should start with a greeting and introduce a question about the topic.",
"Follow up question about the topic",
"Follow up question about the topic",
"Follow up question about the topic",
"Follow up question about the topic"
]
}
try:
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"topic": "topic", "questions": ["question", "question", "question"]}')
'You are a helpful assistant designed to output JSON on this format: ' + str(json_format))
},
{
"role": "user",
"content": (
'Formulate a set of 3 questions of ' + difficulty + ' difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a '
'meaningful discussion on the topic of "' + topic + '". Provide inquiries, ensuring '
'Formulate a set of 5 questions of hard difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a '
'meaningful discussion on the topic of "' + random.choice(
mti_topics) + '". Provide inquiries, ensuring '
'they explore various aspects, perspectives, and implications related to the topic.'
'Make sure that the generated question does not contain forbidden subjects in muslim countries.')
@@ -772,10 +920,22 @@ def grade_speaking_task_3():
"comment": "The audio recorded does not contain enough english words to be graded.",
"overall": 0,
"task_response": {
"Fluency and Coherence": 0,
"Lexical Resource": 0,
"Grammatical Range and Accuracy": 0,
"Pronunciation": 0
"Fluency and Coherence": {
"grade": 0.0,
"comment": ""
},
"Lexical Resource": {
"grade": 0.0,
"comment": ""
},
"Grammatical Range and Accuracy": {
"grade": 0.0,
"comment": ""
},
"Pronunciation": {
"grade": 0.0,
"comment": ""
}
}
}
@@ -801,22 +961,28 @@ def grade_speaking_task_3():
["answer"],
GEN_QUESTION_TEMPERATURE))
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"comment": "comment about answer quality", "overall": 0.0, '
'"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, '
'"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}')
json_format = {
"comment": "extensive comment about answer quality",
"overall": 0.0,
"task_response": {
"Fluency and Coherence": {
"grade": 0.0,
"comment": "extensive comment about fluency and coherence, use examples to justify the grade awarded."
},
"Lexical Resource": {
"grade": 0.0,
"comment": "extensive comment about lexical resource, use examples to justify the grade awarded."
},
"Grammatical Range and Accuracy": {
"grade": 0.0,
"comment": "extensive comment about grammatical range and accuracy, use examples to justify the grade awarded."
},
"Pronunciation": {
"grade": 0.0,
"comment": "extensive comment about pronunciation on the transcribed answer, use examples to justify the grade awarded."
}
}
}
]
message = (
"Evaluate the given Speaking Part 3 response based on the IELTS grading system, ensuring a "
"strict assessment that penalizes errors. Deduct points for deviations from the task, and "
"assign a score of 0 if the response fails to address the question. Additionally, provide detailed "
"commentary highlighting both strengths and weaknesses in the response."
"\n\n The questions and answers are: \n\n'")
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Formatting answers and questions for prompt.")
formatted_text = ""
@@ -826,17 +992,41 @@ def grade_speaking_task_3():
logging.info("POST - speaking_task_3 - " + str(
request_id) + " - Formatted answers and questions for prompt: " + formatted_text)
message += formatted_text
grade_message = (
"Evaluate the given Speaking Part 3 response based on the IELTS grading system, ensuring a "
"strict assessment that penalizes errors. Deduct points for deviations from the task, and "
"assign a score of 0 if the response fails to address the question. Additionally, provide detailed "
"commentary highlighting both strengths and weaknesses in the response."
"\n\n The questions and answers are: \n\n'")
messages.append({
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: ' + str(json_format))
},
{
"role": "user",
"content": message
})
"content": grade_message
},
{
"role": "user",
"content": 'Address the student as "you" and pay special attention to coherence between the answers.'
},
{
"role": "user",
"content": 'For pronunciations act as if you heard the answers and they were transcripted as you heard them.'
},
{
"role": "user",
"content": 'The comments must be long, detailed, justify the grading and suggest improvements.'
}
]
token_count = count_total_tokens(messages)
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Requesting grading of the answers.")
response = make_openai_call(GPT_3_5_TURBO, messages, token_count, ["comment"], GRADING_TEMPERATURE)
response = make_openai_call(GPT_4_O, messages, token_count, ["comment"], GRADING_TEMPERATURE)
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Answers graded: " + str(response))
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Adding perfect answers to response.")

23
helper/heygen_api.py
View File

@@ -29,26 +29,32 @@ GET_HEADER = {
def create_videos_and_save_to_db(exercises, template, id):
avatar = random.choice(list(AvatarEnum))
# Speaking 1
# Using list comprehension to find the element with the desired value in the 'type' field
found_exercises_1 = [element for element in exercises if element.get('type') == 1]
# Check if any elements were found
if found_exercises_1:
exercise_1 = found_exercises_1[0]
sp1_questions = []
app.app.logger.info('Creating video for speaking part 1')
sp1_result = create_video(exercise_1["question"], random.choice(list(AvatarEnum)))
for question in exercise_1["questions"]:
sp1_result = create_video(question, avatar)
if sp1_result is not None:
sound_file_path = VIDEO_FILES_PATH + sp1_result
firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp1_result
url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
sp1_video_path = firebase_file_path
sp1_video_url = url
template["exercises"][0]["text"] = exercise_1["question"]
template["exercises"][0]["title"] = exercise_1["topic"]
template["exercises"][0]["video_url"] = sp1_video_url
template["exercises"][0]["video_path"] = sp1_video_path
video = {
"text": question,
"video_path": firebase_file_path,
"video_url": url
}
sp1_questions.append(video)
else:
app.app.logger.error("Failed to create video for part 1 question: " + exercise_1["question"])
template["exercises"][0]["prompts"] = sp1_questions
template["exercises"][0]["first_title"] = exercise_1["first_topic"]
template["exercises"][0]["second_title"] = exercise_1["second_topic"]
# Speaking 2
# Using list comprehension to find the element with the desired value in the 'type' field
@@ -57,7 +63,7 @@ def create_videos_and_save_to_db(exercises, template, id):
if found_exercises_2:
exercise_2 = found_exercises_2[0]
app.app.logger.info('Creating video for speaking part 2')
sp2_result = create_video(exercise_2["question"], random.choice(list(AvatarEnum)))
sp2_result = create_video(exercise_2["question"], avatar)
if sp2_result is not None:
sound_file_path = VIDEO_FILES_PATH + sp2_result
firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp2_result
@@ -79,7 +85,6 @@ def create_videos_and_save_to_db(exercises, template, id):
if found_exercises_3:
exercise_3 = found_exercises_3[0]
sp3_questions = []
avatar = random.choice(list(AvatarEnum))
app.app.logger.info('Creating videos for speaking part 3')
for question in exercise_3["questions"]:
result = create_video(question, avatar)

11
helper/question_templates.py
View File

@@ -1136,12 +1136,11 @@ def getSpeakingTemplate():
"exercises": [
{
"id": str(uuid.uuid4()),
"prompts": [],
"text": "text",
"title": "topic",
"video_url": "sp1_video_url",
"video_path": "sp1_video_path",
"type": "speaking"
"prompts": ["questions"],
"text": "Listen carefully and respond.",
"first_title": "first_topic",
"second_title": "second_topic",
"type": "interactiveSpeaking"
},
{
"id": str(uuid.uuid4()),