devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve grading to be more strict and give 0 if the question is not addressed.

Browse Source
This commit is contained in:
Cristiano Ferreira
2024-01-23 23:23:17 +00:00
parent 64a4759fbc
commit bc2cedb821
1 changed files with 46 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
app.py
View File

@@ -207,14 +207,15 @@ def grade_writing_task_1():
question = data.get('question') question = data.get('question')
answer = data.get('answer') answer = data.get('answer')
if has_words(answer): if has_words(answer):
message = ( message = ("Evaluate the given Writing Task 1 response based on the IELTS grading system, ensuring a "
"Grade this Writing Task 1 answer according to ielts grading system and provide an example of a perfect " "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
"answer (min 150 words) and an elaborated comment where you deep dive into what is wrong and right" "assign a score of 0 if the response fails to address the question. Additionally, provide an "
" about the answer. Provide your answer on the following json format: {'perfect_answer': 'example " "exemplary answer with a minimum of 150 words, along with a detailed commentary highlighting "
"perfect answer', 'comment': 'comment about answer quality', 'overall': 7.0, " "both strengths and weaknesses in the response. Present your evaluation in JSON format with "
"'task_response': {'Task Achievement': 0.0, 'Coherence and Cohesion': 0.0, 'Lexical Resource': 0.0, " "the following structure: {'perfect_answer': 'example perfect answer', 'comment': "
"'Grammatical Range and Accuracy': 0.0}}\n The question was '" + question + "' " "'comment about answer quality', 'overall': 0.0, 'task_response': {'Task Achievement': 0.0, "
"and the answer was '" + answer + "'") "'Coherence and Cohesion': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range and Accuracy': "
"0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
token_count = count_tokens(message)["n_tokens"] token_count = count_tokens(message)["n_tokens"]
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
["comment"], ["comment"],
@@ -261,14 +262,15 @@ def grade_writing_task_2():
question = data.get('question') question = data.get('question')
answer = data.get('answer') answer = data.get('answer')
if has_words(answer): if has_words(answer):
message = ( message = ("Evaluate the given Writing Task 2 response based on the IELTS grading system, ensuring a "
"Grade this Writing Task 2 answer according to ielts grading system and provide an example of a perfect " "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
"answer (min 250 words) and an elaborated comment where you deep dive into what is wrong and right " "assign a score of 0 if the response fails to address the question. Additionally, provide an "
"about the answer. Provide your answer on the following json format: {'perfect_answer': 'example " "exemplary answer with a minimum of 250 words, along with a detailed commentary highlighting "
"perfect answer', 'comment': 'comment about answer quality', 'overall': 7.0, " "both strengths and weaknesses in the response. Present your evaluation in JSON format with "
"'task_response': {'Task Achievement': 0.0, 'Coherence and Cohesion': 0.0, 'Lexical Resource': 0.0, " "the following structure: {'perfect_answer': 'example perfect answer', 'comment': "
"'Grammatical Range and Accuracy': 0.0}}\n The question was '" + question + "' " "'comment about answer quality', 'overall': 0.0, 'task_response': {'Task Achievement': 0.0, "
"and the answer was '" + answer + "'") "'Coherence and Cohesion': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range and Accuracy': "
"0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
token_count = count_tokens(message)["n_tokens"] token_count = count_tokens(message)["n_tokens"]
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
["comment"], ["comment"],
@@ -308,6 +310,7 @@ def get_writing_task_2_general_question():
except Exception as e: except Exception as e:
return str(e) return str(e)
# THE SAVING OF WRITING IS DONE WITHOUT THE API ON THE FRONTEND # THE SAVING OF WRITING IS DONE WITHOUT THE API ON THE FRONTEND
# @app.route('/writing', methods=['POST']) # @app.route('/writing', methods=['POST'])
# @jwt_required() # @jwt_required()
@@ -342,13 +345,14 @@ def grade_speaking_task_1():
download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name) download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
answer = speech_to_text(sound_file_name) answer = speech_to_text(sound_file_name)
if has_10_words(answer): if has_10_words(answer):
message = ( message = ("Evaluate the given Speaking Part 1 response based on the IELTS grading system, ensuring a "
"Grade this Speaking Part 1 answer according to ielts grading system and provide an elaborated " "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
"comment where you deep dive into what is wrong and right about the answer." "assign a score of 0 if the response fails to address the question. Additionally, provide "
"Please assign a grade of 0 if the answer provided does not address the question." "detailed commentary highlighting both strengths and weaknesses in the response. Present your "
"Provide your answer on the following json format: {'comment': 'comment about answer quality', 'overall': 0.0, " "evaluation in JSON format with "
"'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range and Accuracy': 0.0, " "the following structure: {'comment': 'comment about answer quality', 'overall': 0.0, "
"'Pronunciation': 0.0}}\n The question was '" + question + "' and the answer was '" + answer + "'") "'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range "
"and Accuracy': 0.0, 'Pronunciation': 0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
token_count = count_tokens(message)["n_tokens"] token_count = count_tokens(message)["n_tokens"]
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
["comment"], ["comment"],
@@ -411,13 +415,15 @@ def grade_speaking_task_2():
download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name) download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
answer = speech_to_text(sound_file_name) answer = speech_to_text(sound_file_name)
if has_10_words(answer): if has_10_words(answer):
message = ( message = ("Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a "
"Grade this Speaking Part 2 answer according to ielts grading system and provide an elaborated " "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
"comment where you deep dive into what is wrong and right about the answer." "assign a score of 0 if the response fails to address the question. Additionally, provide "
"Please assign a grade of 0 if the answer provided does not address the question." "detailed commentary highlighting both strengths and weaknesses in the response. Present your "
"Provide your answer on the following json format: {'comment': 'comment about answer quality', 'overall': 0.0, " "evaluation in JSON format with "
"'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range and Accuracy': 0.0, " "the following structure: {'comment': 'comment about answer quality', 'overall': 0.0, "
"'Pronunciation': 0.0}}\n The question was '" + question + "' and the answer was '" + answer + "'") "'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range "
"and Accuracy': 0.0, "
"'Pronunciation': 0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
token_count = count_tokens(message)["n_tokens"] token_count = count_tokens(message)["n_tokens"]
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
["comment"], ["comment"],
@@ -525,9 +531,10 @@ def grade_speaking_task_3():
None, None,
GEN_QUESTION_TEMPERATURE)) GEN_QUESTION_TEMPERATURE))
message = ( message = (
"Grade this Speaking Part 3 answer according to ielts grading system and provide " "Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a "
"an elaborated comment where you deep dive into what is wrong and right about the answers." "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
"Please assign a grade of 0 if the answer provided does not address the question." "assign a score of 0 if the response fails to address the question. Additionally, provide detailed "
"commentary highlighting both strengths and weaknesses in the response."
"\n\n The questions and answers are: \n\n'") "\n\n The questions and answers are: \n\n'")
formatted_text = "" formatted_text = ""
@@ -587,6 +594,7 @@ def save_speaking():
except Exception as e: except Exception as e:
return str(e) return str(e)
@app.route("/speaking/generate_speaking_video", methods=['POST']) @app.route("/speaking/generate_speaking_video", methods=['POST'])
@jwt_required() @jwt_required()
def generate_speaking_video(): def generate_speaking_video():
@@ -602,7 +610,7 @@ def generate_speaking_video():
return { return {
"text": data["question"], "text": data["question"],
"prompts": data["prompts"] if "prompts" in data else [], "prompts": data["prompts"] if "prompts" in data else [],
"title": data["topic"], "title": data["topic"],
"video_url": sp1_video_url, "video_url": sp1_video_url,
"video_path": sp1_video_path, "video_path": sp1_video_path,
@@ -612,10 +620,11 @@ def generate_speaking_video():
else: else:
app.logger.error("Failed to create video for part 1 question: " + data["question"]) app.logger.error("Failed to create video for part 1 question: " + data["question"])
return str("Failed to create video for part 1 question: " + data["question"]) return str("Failed to create video for part 1 question: " + data["question"])
except Exception as e: except Exception as e:
return str(e) return str(e)
@app.route("/speaking/generate_interactive_video", methods=['POST']) @app.route("/speaking/generate_interactive_video", methods=['POST'])
@jwt_required() @jwt_required()
def generate_interactive_video(): def generate_interactive_video():
@@ -645,10 +654,11 @@ def generate_interactive_video():
"title": data["topic"], "title": data["topic"],
"type": "interactiveSpeaking", "type": "interactiveSpeaking",
"id": uuid.uuid4() "id": uuid.uuid4()
} }
except Exception as e: except Exception as e:
return str(e) return str(e)
@app.route('/reading_passage_1', methods=['GET']) @app.route('/reading_passage_1', methods=['GET'])
@jwt_required() @jwt_required()
def get_reading_passage_1_question(): def get_reading_passage_1_question():