From 7049fd86d48facedd98078865b14d6bb1bf49b20 Mon Sep 17 00:00:00 2001 From: Cristiano Ferreira Date: Sun, 24 Mar 2024 01:33:21 +0000 Subject: [PATCH] Improve speaking grading --- app.py | 70 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/app.py b/app.py index db5f911..c49a20f 100644 --- a/app.py +++ b/app.py @@ -97,7 +97,8 @@ def get_listening_section_2_question(): app.logger.info("Generated monologue: " + str(monologue)) start_id = 11 - exercises = generate_listening_monologue_exercises(monologue, req_exercises, number_of_exercises_q, start_id, difficulty) + exercises = generate_listening_monologue_exercises(monologue, req_exercises, number_of_exercises_q, start_id, + difficulty) return { "exercises": exercises, "text": monologue, @@ -158,7 +159,8 @@ def get_listening_section_4_question(): app.logger.info("Generated monologue: " + str(monologue)) start_id = 31 - exercises = generate_listening_monologue_exercises(monologue, req_exercises, number_of_exercises_q, start_id, difficulty) + exercises = generate_listening_monologue_exercises(monologue, req_exercises, number_of_exercises_q, start_id, + difficulty) return { "exercises": exercises, "text": monologue, @@ -257,9 +259,9 @@ def get_writing_task_1_general_question(): try: gen_wt1_question = "Craft a prompt for an IELTS Writing Task 1 General Training exercise that instructs the " \ "student to compose a letter. The prompt should present a specific scenario or situation, " \ - "based on the topic of '" + topic + "', "\ - "requiring the student to provide information, advice, or instructions within the letter. " \ - "Make sure that the generated prompt is of " + difficulty + " difficulty and does not contain forbidden subjects in muslim countries." + "based on the topic of '" + topic + "', " \ + "requiring the student to provide information, advice, or instructions within the letter. " \ + "Make sure that the generated prompt is of " + difficulty + " difficulty and does not contain forbidden subjects in muslim countries." token_count = count_tokens(gen_wt1_question)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_wt1_question, token_count, None, GEN_QUESTION_TEMPERATURE) @@ -317,10 +319,10 @@ def get_writing_task_2_general_question(): topic = request.args.get("topic", default=random.choice(mti_topics)) try: gen_wt2_question = "Craft a comprehensive question of " + difficulty + " difficulty for IELTS Writing Task 2 General Training that directs the candidate " \ - "to delve into an in-depth analysis of contrasting perspectives on the topic of '" + topic + "'. The candidate " \ - "should be asked to discuss the strengths and weaknesses of both viewpoints, provide evidence or " \ - "examples, and present a well-rounded argument before concluding with their personal opinion on the " \ - "subject." + "to delve into an in-depth analysis of contrasting perspectives on the topic of '" + topic + "'. The candidate " \ + "should be asked to discuss the strengths and weaknesses of both viewpoints, provide evidence or " \ + "examples, and present a well-rounded argument before concluding with their personal opinion on the " \ + "subject." token_count = count_tokens(gen_wt2_question)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_wt2_question, token_count, None, GEN_QUESTION_TEMPERATURE) @@ -378,7 +380,7 @@ def grade_speaking_task_1(): token_count = count_tokens(message)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, ["comment"], - GEN_QUESTION_TEMPERATURE) + GRADING_TEMPERATURE) perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following " "Speaking Part 1 question: '" + question + "'") token_count = count_tokens(perfect_answer_message)["n_tokens"] @@ -389,6 +391,11 @@ def grade_speaking_task_1(): GEN_QUESTION_TEMPERATURE) response['transcript'] = answer response['fixed_text'] = get_speaking_corrections(answer) + if response["overall"] == "0.0" or response["overall"] == 0.0: + response["overall"] = round((response["task_response"]["Fluency and Coherence"] + + response["task_response"]["Lexical Resource"] + response["task_response"][ + "Grammatical Range and Accuracy"] + response["task_response"][ + "Pronunciation"]) / 4, 1) return response else: return { @@ -413,10 +420,10 @@ def get_speaking_task_1_question(): topic = request.args.get("topic", default=random.choice(mti_topics)) try: gen_sp1_question = "Craft a thought-provoking question of " + difficulty + " difficulty for IELTS Speaking Part 1 that encourages candidates to delve deeply " \ - "into personal experiences, preferences, or insights on the topic of '" + topic + "'. Instruct the candidate to offer " \ - "not only detailed descriptions but also provide nuanced explanations, examples, or anecdotes to enrich " \ - "their response. Make sure that the generated question does not contain forbidden subjects in muslim countries." \ - "Provide your response in this json format: {'topic': 'topic','question': 'question'}" + "into personal experiences, preferences, or insights on the topic of '" + topic + "'. Instruct the candidate to offer " \ + "not only detailed descriptions but also provide nuanced explanations, examples, or anecdotes to enrich " \ + "their response. Make sure that the generated question does not contain forbidden subjects in muslim countries." \ + "Provide your response in this json format: {'topic': 'topic','question': 'question'}" token_count = count_tokens(gen_sp1_question)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp1_question, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE) @@ -453,7 +460,7 @@ def grade_speaking_task_2(): token_count = count_tokens(message)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, ["comment"], - GEN_QUESTION_TEMPERATURE) + GRADING_TEMPERATURE) perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following " "Speaking Part 2 question: '" + question + "'") token_count = count_tokens(perfect_answer_message)["n_tokens"] @@ -464,6 +471,12 @@ def grade_speaking_task_2(): GEN_QUESTION_TEMPERATURE) response['transcript'] = answer response['fixed_text'] = get_speaking_corrections(answer) + if response["overall"] == "0.0" or response["overall"] == 0.0: + response["overall"] = round((response["task_response"]["Fluency and Coherence"] + + response["task_response"]["Lexical Resource"] + response["task_response"][ + "Grammatical Range and Accuracy"] + response["task_response"][ + "Pronunciation"]) / 4, 1) + return response else: return { @@ -488,11 +501,11 @@ def get_speaking_task_2_question(): topic = request.args.get("topic", default=random.choice(mti_topics)) try: gen_sp2_question = "Create a question of " + difficulty + " difficulty for IELTS Speaking Part 2 that encourages candidates to narrate a personal experience " \ - "or story related to the topic of '" + topic + "'. Include 3 prompts that guide the candidate to describe " \ - "specific aspects of the experience, such as details about the situation, their actions, and the " \ - "reasons it left a lasting impression. Make sure that the generated question does not contain forbidden subjects in muslim countries." \ - "Provide your response in this json format: {'topic': 'topic','question': 'question', " \ - "'prompts': ['prompt_1', 'prompt_2', 'prompt_3']}" + "or story related to the topic of '" + topic + "'. Include 3 prompts that guide the candidate to describe " \ + "specific aspects of the experience, such as details about the situation, their actions, and the " \ + "reasons it left a lasting impression. Make sure that the generated question does not contain forbidden subjects in muslim countries." \ + "Provide your response in this json format: {'topic': 'topic','question': 'question', " \ + "'prompts': ['prompt_1', 'prompt_2', 'prompt_3']}" token_count = count_tokens(gen_sp2_question)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp2_question, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE) @@ -511,11 +524,11 @@ def get_speaking_task_3_question(): topic = request.args.get("topic", default=random.choice(mti_topics)) try: gen_sp3_question = "Formulate a set of 3 questions of " + difficulty + " difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a " \ - "meaningful discussion on the topic of '" + topic + "'. Provide inquiries, ensuring " \ - "they explore various aspects, perspectives, and implications related to the topic. " \ - "Make sure that the generated question does not contain forbidden subjects in muslim countries." \ - "Provide your response in this json format: {'topic': 'topic','questions': ['question', " \ - "'question', 'question']}" + "meaningful discussion on the topic of '" + topic + "'. Provide inquiries, ensuring " \ + "they explore various aspects, perspectives, and implications related to the topic. " \ + "Make sure that the generated question does not contain forbidden subjects in muslim countries." \ + "Provide your response in this json format: {'topic': 'topic','questions': ['question', " \ + "'question', 'question']}" token_count = count_tokens(gen_sp3_question)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp3_question, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE) @@ -586,13 +599,18 @@ def grade_speaking_task_3(): token_count = count_tokens(message)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, ["comment"], - GEN_QUESTION_TEMPERATURE) + GRADING_TEMPERATURE) for i, answer in enumerate(perfect_answers, start=1): response['perfect_answer_' + str(i)] = answer for i, answer in enumerate(text_answers, start=1): response['transcript_' + str(i)] = answer response['fixed_text_' + str(i)] = get_speaking_corrections(answer) + if response["overall"] == "0.0" or response["overall"] == 0.0: + response["overall"] = round((response["task_response"]["Fluency and Coherence"] + response["task_response"][ + "Lexical Resource"] + response["task_response"]["Grammatical Range and Accuracy"] + + response["task_response"]["Pronunciation"]) / 4, 1) + return response except Exception as e: return str(e), 400