devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speaking on api latest version.

Browse Source
This commit is contained in:
Cristiano Ferreira
2024-05-20 15:24:05 +01:00
parent 9654d9ff64
commit a0a193844d
1 changed files with 149 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
app.py
View File

@@ -473,7 +473,7 @@ def grade_speaking_task_1():
response['perfect_answer'] = make_openai_call(GPT_3_5_TURBO, response['perfect_answer'] = make_openai_call(GPT_3_5_TURBO,
perfect_answer_messages, perfect_answer_messages,
token_count, token_count,
None, ["answer"],
GEN_QUESTION_TEMPERATURE)["answer"] GEN_QUESTION_TEMPERATURE)["answer"]
logging.info("POST - speaking_task_1 - " + str( logging.info("POST - speaking_task_1 - " + str(
request_id) + " - Perfect answer: " + response['perfect_answer']) request_id) + " - Perfect answer: " + response['perfect_answer'])
@@ -516,14 +516,32 @@ def get_speaking_task_1_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties)) difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics)) topic = request.args.get("topic", default=random.choice(mti_topics))
try: try:
gen_sp1_question = "Craft a thought-provoking question of " + difficulty + " difficulty for IELTS Speaking Part 1 that encourages candidates to delve deeply " \ messages = [
"into personal experiences, preferences, or insights on the topic of '" + topic + "'. Instruct the candidate to offer " \ {
"not only detailed descriptions but also provide nuanced explanations, examples, or anecdotes to enrich " \ "role": "system",
"their response. Make sure that the generated question does not contain forbidden subjects in muslim countries." \ "content": (
"Provide your response in this json format: {'topic': 'topic','question': 'question'}" 'You are a helpful assistant designed to output JSON on this format: '
token_count = count_tokens(gen_sp1_question)["n_tokens"] '{"topic": "topic", "question": "question"}')
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp1_question, token_count, GEN_FIELDS, },
GEN_QUESTION_TEMPERATURE) {
"role": "user",
"content": (
'Craft a thought-provoking question of ' + difficulty + ' difficulty for IELTS Speaking Part 1 '
'that encourages candidates to delve deeply into '
'personal experiences, preferences, or insights on the topic '
'of "' + topic + '". Instruct the candidate '
'to offer not only detailed '
'descriptions but also provide '
'nuanced explanations, examples, '
'or anecdotes to enrich their response. '
'Make sure that the generated question '
'does not contain forbidden subjects in '
'muslim countries.')
}
]
token_count = count_total_tokens(messages)
response = make_openai_call(GPT_4_O, messages, token_count, ["topic"],
GEN_QUESTION_TEMPERATURE)
response["type"] = 1 response["type"] = 1
response["difficulty"] = difficulty response["difficulty"] = difficulty
response["topic"] = topic response["topic"] = topic
@@ -554,33 +572,53 @@ def grade_speaking_task_2():
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Transcripted answer: " + answer) logging.info("POST - speaking_task_2 - " + str(request_id) + " - Transcripted answer: " + answer)
if has_x_words(answer, 20): if has_x_words(answer, 20):
message = ("Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a " messages = [
"strict assessment that penalizes errors. Deduct points for deviations from the task, and " {
"assign a score of 0 if the response fails to address the question. Additionally, provide " "role": "system",
"detailed commentary highlighting both strengths and weaknesses in the response. Present your " "content": (
"evaluation in JSON format with " 'You are a helpful assistant designed to output JSON on this format: '
"the following structure: {'comment': 'comment about answer quality', 'overall': 0.0, " '{"comment": "comment about answer quality", "overall": 0.0, '
"'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range " '"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, '
"and Accuracy': 0.0, " '"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}')
"'Pronunciation': 0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'") },
token_count = count_tokens(message)["n_tokens"] {
"role": "user",
"content": (
'Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a '
'strict assessment that penalizes errors. Deduct points for deviations from the task, and '
'assign a score of 0 if the response fails to address the question. Additionally, provide '
'detailed commentary highlighting both strengths and weaknesses in the response.'
'\n Question: "' + question + '" \n Answer: "' + answer + '"')
}
]
token_count = count_total_tokens(messages)
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Requesting grading of the answer.") logging.info("POST - speaking_task_2 - " + str(request_id) + " - Requesting grading of the answer.")
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, response = make_openai_call(GPT_3_5_TURBO, messages, token_count,["comment"],
["comment"],
GRADING_TEMPERATURE) GRADING_TEMPERATURE)
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Answer graded: " + str(response)) logging.info("POST - speaking_task_2 - " + str(request_id) + " - Answer graded: " + str(response))
perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following " perfect_answer_messages = [
"Speaking Part 2 question: '" + question + "'") {
token_count = count_tokens(perfect_answer_message)["n_tokens"] "role": "system",
"content": ('You are a helpful assistant designed to output JSON on this format: '
'{"answer": "perfect answer"}')
},
{
"role": "user",
"content": (
'Provide a perfect answer according to ielts grading system to the following '
'Speaking Part 2 question: "' + question + '"')
}
]
token_count = count_total_tokens(perfect_answer_messages)
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Requesting perfect answer.") logging.info("POST - speaking_task_2 - " + str(request_id) + " - Requesting perfect answer.")
response['perfect_answer'] = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, response['perfect_answer'] = make_openai_call(GPT_3_5_TURBO,
perfect_answer_message, perfect_answer_messages,
token_count, token_count,
None, ["answer"],
GEN_QUESTION_TEMPERATURE) GEN_QUESTION_TEMPERATURE)["answer"]
logging.info("POST - speaking_task_2 - " + str( logging.info("POST - speaking_task_2 - " + str(
request_id) + " - Perfect answer: " + response['perfect_answer']) request_id) + " - Perfect answer: " + response['perfect_answer'])
@@ -622,15 +660,31 @@ def get_speaking_task_2_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties)) difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics)) topic = request.args.get("topic", default=random.choice(mti_topics))
try: try:
gen_sp2_question = "Create a question of " + difficulty + " difficulty for IELTS Speaking Part 2 that encourages candidates to narrate a personal experience " \ messages = [
"or story related to the topic of '" + topic + "'. Include 3 prompts that guide the candidate to describe " \ {
"specific aspects of the experience, such as details about the situation, their actions, and the " \ "role": "system",
"reasons it left a lasting impression. Make sure that the generated question does not contain forbidden subjects in muslim countries." \ "content": (
"Provide your response in this json format: {'topic': 'topic','question': 'question', " \ 'You are a helpful assistant designed to output JSON on this format: '
"'prompts': ['prompt_1', 'prompt_2', 'prompt_3']}" '{"topic": "topic", "question": "question"}')
token_count = count_tokens(gen_sp2_question)["n_tokens"] },
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp2_question, token_count, GEN_FIELDS, {
GEN_QUESTION_TEMPERATURE) "role": "user",
"content": (
'Craft a thought-provoking question of ' + difficulty + ' difficulty for IELTS Speaking Part 2 '
'that encourages candidates to narrate a '
'personal experience or story related to the topic '
'of "' + topic + '". Include 3 prompts that '
'guide the candidate to describe '
'specific aspects of the experience, '
'such as details about the situation, '
'their actions, and the reasons it left a '
'lasting impression. Make sure that the '
'generated question does not contain '
'forbidden subjects in muslim countries.')
}
]
token_count = count_total_tokens(messages)
response = make_openai_call(GPT_4_O, messages, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE)
response["type"] = 2 response["type"] = 2
response["difficulty"] = difficulty response["difficulty"] = difficulty
response["topic"] = topic response["topic"] = topic
@@ -645,15 +699,25 @@ def get_speaking_task_3_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties)) difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics)) topic = request.args.get("topic", default=random.choice(mti_topics))
try: try:
gen_sp3_question = "Formulate a set of 3 questions of " + difficulty + " difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a " \ messages = [
"meaningful discussion on the topic of '" + topic + "'. Provide inquiries, ensuring " \ {
"they explore various aspects, perspectives, and implications related to the topic. " \ "role": "system",
"Make sure that the generated question does not contain forbidden subjects in muslim countries." \ "content": (
"Provide your response in this json format: {'topic': 'topic','questions': ['question', " \ 'You are a helpful assistant designed to output JSON on this format: '
"'question', 'question']}" '{"topic": "topic", "questions": ["question", "question", "question"]}')
token_count = count_tokens(gen_sp3_question)["n_tokens"] },
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp3_question, token_count, GEN_FIELDS, {
GEN_QUESTION_TEMPERATURE) "role": "user",
"content": (
'Formulate a set of 3 questions of ' + difficulty + ' difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a '
'meaningful discussion on the topic of "' + topic + '". Provide inquiries, ensuring '
'they explore various aspects, perspectives, and implications related to the topic.'
'Make sure that the generated question does not contain forbidden subjects in muslim countries.')
}
]
token_count = count_total_tokens(messages)
response = make_openai_call(GPT_4_O, messages, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE)
# Remove the numbers from the questions only if the string starts with a number # Remove the numbers from the questions only if the string starts with a number
response["questions"] = [re.sub(r"^\d+\.\s*", "", question) if re.match(r"^\d+\.", question) else question for response["questions"] = [re.sub(r"^\d+\.\s*", "", question) if re.match(r"^\d+\.", question) else question for
question in response["questions"]] question in response["questions"]]
@@ -706,16 +770,39 @@ def grade_speaking_task_3():
"Pronunciation": 0 "Pronunciation": 0
} }
} }
perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following "
"Speaking Part 3 question: '" + item["question"] + "'") perfect_answer_messages = [
token_count = count_tokens(perfect_answer_message)["n_tokens"] {
"role": "system",
"content": ('You are a helpful assistant designed to output JSON on this format: '
'{"answer": "perfect answer"}')
},
{
"role": "user",
"content": (
'Provide a perfect answer according to ielts grading system to the following '
'Speaking Part 3 question: "' + item["question"] + '"')
}
]
token_count = count_total_tokens(perfect_answer_messages)
logging.info("POST - speaking_task_3 - " + str( logging.info("POST - speaking_task_3 - " + str(
request_id) + " - Requesting perfect answer for question: " + item["question"]) request_id) + " - Requesting perfect answer for question: " + item["question"])
perfect_answers.append(make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, perfect_answers.append(make_openai_call(GPT_3_5_TURBO,
perfect_answer_message, perfect_answer_messages,
token_count, token_count,
None, ["answer"],
GEN_QUESTION_TEMPERATURE)) GEN_QUESTION_TEMPERATURE))
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"comment": "comment about answer quality", "overall": 0.0, '
'"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, '
'"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}')
}
]
message = ( message = (
"Evaluate the given Speaking Part 3 response based on the IELTS grading system, ensuring a " "Evaluate the given Speaking Part 3 response based on the IELTS grading system, ensuring a "
"strict assessment that penalizes errors. Deduct points for deviations from the task, and " "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
@@ -732,17 +819,16 @@ def grade_speaking_task_3():
request_id) + " - Formatted answers and questions for prompt: " + formatted_text) request_id) + " - Formatted answers and questions for prompt: " + formatted_text)
message += formatted_text message += formatted_text
message += (
"'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', "
"'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, "
"'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}")
token_count = count_tokens(message)["n_tokens"] messages.append({
"role": "user",
"content": message
})
token_count = count_total_tokens(messages)
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Requesting grading of the answers.") logging.info("POST - speaking_task_3 - " + str(request_id) + " - Requesting grading of the answers.")
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, response = make_openai_call(GPT_3_5_TURBO, messages, token_count, ["comment"], GRADING_TEMPERATURE)
["comment"],
GRADING_TEMPERATURE)
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Answers graded: " + str(response)) logging.info("POST - speaking_task_3 - " + str(request_id) + " - Answers graded: " + str(response))
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Adding perfect answers to response.") logging.info("POST - speaking_task_3 - " + str(request_id) + " - Adding perfect answers to response.")