devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speaking on api latest version.

Browse Source
This commit is contained in:
Cristiano Ferreira
2024-05-20 15:24:05 +01:00
parent 9654d9ff64
commit a0a193844d
1 changed files with 149 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
app.py
View File

@@ -473,7 +473,7 @@ def grade_speaking_task_1():
response['perfect_answer'] = make_openai_call(GPT_3_5_TURBO,
perfect_answer_messages,
token_count,
None,
["answer"],
GEN_QUESTION_TEMPERATURE)["answer"]
logging.info("POST - speaking_task_1 - " + str(
request_id) + " - Perfect answer: " + response['perfect_answer'])
@@ -516,13 +516,31 @@ def get_speaking_task_1_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics))
try:
gen_sp1_question = "Craft a thought-provoking question of " + difficulty + " difficulty for IELTS Speaking Part 1 that encourages candidates to delve deeply " \
"into personal experiences, preferences, or insights on the topic of '" + topic + "'. Instruct the candidate to offer " \
"not only detailed descriptions but also provide nuanced explanations, examples, or anecdotes to enrich " \
"their response. Make sure that the generated question does not contain forbidden subjects in muslim countries." \
"Provide your response in this json format: {'topic': 'topic','question': 'question'}"
token_count = count_tokens(gen_sp1_question)["n_tokens"]
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp1_question, token_count, GEN_FIELDS,
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"topic": "topic", "question": "question"}')
},
{
"role": "user",
"content": (
'Craft a thought-provoking question of ' + difficulty + ' difficulty for IELTS Speaking Part 1 '
'that encourages candidates to delve deeply into '
'personal experiences, preferences, or insights on the topic '
'of "' + topic + '". Instruct the candidate '
'to offer not only detailed '
'descriptions but also provide '
'nuanced explanations, examples, '
'or anecdotes to enrich their response. '
'Make sure that the generated question '
'does not contain forbidden subjects in '
'muslim countries.')
}
]
token_count = count_total_tokens(messages)
response = make_openai_call(GPT_4_O, messages, token_count, ["topic"],
GEN_QUESTION_TEMPERATURE)
response["type"] = 1
response["difficulty"] = difficulty
@@ -554,33 +572,53 @@ def grade_speaking_task_2():
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Transcripted answer: " + answer)
if has_x_words(answer, 20):
message = ("Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a "
"strict assessment that penalizes errors. Deduct points for deviations from the task, and "
"assign a score of 0 if the response fails to address the question. Additionally, provide "
"detailed commentary highlighting both strengths and weaknesses in the response. Present your "
"evaluation in JSON format with "
"the following structure: {'comment': 'comment about answer quality', 'overall': 0.0, "
"'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range "
"and Accuracy': 0.0, "
"'Pronunciation': 0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
token_count = count_tokens(message)["n_tokens"]
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"comment": "comment about answer quality", "overall": 0.0, '
'"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, '
'"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}')
},
{
"role": "user",
"content": (
'Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a '
'strict assessment that penalizes errors. Deduct points for deviations from the task, and '
'assign a score of 0 if the response fails to address the question. Additionally, provide '
'detailed commentary highlighting both strengths and weaknesses in the response.'
'\n Question: "' + question + '" \n Answer: "' + answer + '"')
}
]
token_count = count_total_tokens(messages)
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Requesting grading of the answer.")
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
["comment"],
response = make_openai_call(GPT_3_5_TURBO, messages, token_count,["comment"],
GRADING_TEMPERATURE)
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Answer graded: " + str(response))
perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following "
"Speaking Part 2 question: '" + question + "'")
token_count = count_tokens(perfect_answer_message)["n_tokens"]
perfect_answer_messages = [
{
"role": "system",
"content": ('You are a helpful assistant designed to output JSON on this format: '
'{"answer": "perfect answer"}')
},
{
"role": "user",
"content": (
'Provide a perfect answer according to ielts grading system to the following '
'Speaking Part 2 question: "' + question + '"')
}
]
token_count = count_total_tokens(perfect_answer_messages)
logging.info("POST - speaking_task_2 - " + str(request_id) + " - Requesting perfect answer.")
response['perfect_answer'] = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT,
perfect_answer_message,
response['perfect_answer'] = make_openai_call(GPT_3_5_TURBO,
perfect_answer_messages,
token_count,
None,
GEN_QUESTION_TEMPERATURE)
["answer"],
GEN_QUESTION_TEMPERATURE)["answer"]
logging.info("POST - speaking_task_2 - " + str(
request_id) + " - Perfect answer: " + response['perfect_answer'])
@@ -622,15 +660,31 @@ def get_speaking_task_2_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics))
try:
gen_sp2_question = "Create a question of " + difficulty + " difficulty for IELTS Speaking Part 2 that encourages candidates to narrate a personal experience " \
"or story related to the topic of '" + topic + "'. Include 3 prompts that guide the candidate to describe " \
"specific aspects of the experience, such as details about the situation, their actions, and the " \
"reasons it left a lasting impression. Make sure that the generated question does not contain forbidden subjects in muslim countries." \
"Provide your response in this json format: {'topic': 'topic','question': 'question', " \
"'prompts': ['prompt_1', 'prompt_2', 'prompt_3']}"
token_count = count_tokens(gen_sp2_question)["n_tokens"]
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp2_question, token_count, GEN_FIELDS,
GEN_QUESTION_TEMPERATURE)
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"topic": "topic", "question": "question"}')
},
{
"role": "user",
"content": (
'Craft a thought-provoking question of ' + difficulty + ' difficulty for IELTS Speaking Part 2 '
'that encourages candidates to narrate a '
'personal experience or story related to the topic '
'of "' + topic + '". Include 3 prompts that '
'guide the candidate to describe '
'specific aspects of the experience, '
'such as details about the situation, '
'their actions, and the reasons it left a '
'lasting impression. Make sure that the '
'generated question does not contain '
'forbidden subjects in muslim countries.')
}
]
token_count = count_total_tokens(messages)
response = make_openai_call(GPT_4_O, messages, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE)
response["type"] = 2
response["difficulty"] = difficulty
response["topic"] = topic
@@ -645,15 +699,25 @@ def get_speaking_task_3_question():
difficulty = request.args.get("difficulty", default=random.choice(difficulties))
topic = request.args.get("topic", default=random.choice(mti_topics))
try:
gen_sp3_question = "Formulate a set of 3 questions of " + difficulty + " difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a " \
"meaningful discussion on the topic of '" + topic + "'. Provide inquiries, ensuring " \
"they explore various aspects, perspectives, and implications related to the topic. " \
"Make sure that the generated question does not contain forbidden subjects in muslim countries." \
"Provide your response in this json format: {'topic': 'topic','questions': ['question', " \
"'question', 'question']}"
token_count = count_tokens(gen_sp3_question)["n_tokens"]
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp3_question, token_count, GEN_FIELDS,
GEN_QUESTION_TEMPERATURE)
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"topic": "topic", "questions": ["question", "question", "question"]}')
},
{
"role": "user",
"content": (
'Formulate a set of 3 questions of ' + difficulty + ' difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a '
'meaningful discussion on the topic of "' + topic + '". Provide inquiries, ensuring '
'they explore various aspects, perspectives, and implications related to the topic.'
'Make sure that the generated question does not contain forbidden subjects in muslim countries.')
}
]
token_count = count_total_tokens(messages)
response = make_openai_call(GPT_4_O, messages, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE)
# Remove the numbers from the questions only if the string starts with a number
response["questions"] = [re.sub(r"^\d+\.\s*", "", question) if re.match(r"^\d+\.", question) else question for
question in response["questions"]]
@@ -706,16 +770,39 @@ def grade_speaking_task_3():
"Pronunciation": 0
}
}
perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following "
"Speaking Part 3 question: '" + item["question"] + "'")
token_count = count_tokens(perfect_answer_message)["n_tokens"]
perfect_answer_messages = [
{
"role": "system",
"content": ('You are a helpful assistant designed to output JSON on this format: '
'{"answer": "perfect answer"}')
},
{
"role": "user",
"content": (
'Provide a perfect answer according to ielts grading system to the following '
'Speaking Part 3 question: "' + item["question"] + '"')
}
]
token_count = count_total_tokens(perfect_answer_messages)
logging.info("POST - speaking_task_3 - " + str(
request_id) + " - Requesting perfect answer for question: " + item["question"])
perfect_answers.append(make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT,
perfect_answer_message,
perfect_answers.append(make_openai_call(GPT_3_5_TURBO,
perfect_answer_messages,
token_count,
None,
["answer"],
GEN_QUESTION_TEMPERATURE))
messages = [
{
"role": "system",
"content": (
'You are a helpful assistant designed to output JSON on this format: '
'{"comment": "comment about answer quality", "overall": 0.0, '
'"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, '
'"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}')
}
]
message = (
"Evaluate the given Speaking Part 3 response based on the IELTS grading system, ensuring a "
"strict assessment that penalizes errors. Deduct points for deviations from the task, and "
@@ -732,17 +819,16 @@ def grade_speaking_task_3():
request_id) + " - Formatted answers and questions for prompt: " + formatted_text)
message += formatted_text
message += (
"'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', "
"'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, "
"'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}")
token_count = count_tokens(message)["n_tokens"]
messages.append({
"role": "user",
"content": message
})
token_count = count_total_tokens(messages)
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Requesting grading of the answers.")
response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
["comment"],
GRADING_TEMPERATURE)
response = make_openai_call(GPT_3_5_TURBO, messages, token_count, ["comment"], GRADING_TEMPERATURE)
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Answers graded: " + str(response))
logging.info("POST - speaking_task_3 - " + str(request_id) + " - Adding perfect answers to response.")