diff --git a/app.py b/app.py index 379f0fa..3ae82b8 100644 --- a/app.py +++ b/app.py @@ -473,7 +473,7 @@ def grade_speaking_task_1(): response['perfect_answer'] = make_openai_call(GPT_3_5_TURBO, perfect_answer_messages, token_count, - None, + ["answer"], GEN_QUESTION_TEMPERATURE)["answer"] logging.info("POST - speaking_task_1 - " + str( request_id) + " - Perfect answer: " + response['perfect_answer']) @@ -516,14 +516,32 @@ def get_speaking_task_1_question(): difficulty = request.args.get("difficulty", default=random.choice(difficulties)) topic = request.args.get("topic", default=random.choice(mti_topics)) try: - gen_sp1_question = "Craft a thought-provoking question of " + difficulty + " difficulty for IELTS Speaking Part 1 that encourages candidates to delve deeply " \ - "into personal experiences, preferences, or insights on the topic of '" + topic + "'. Instruct the candidate to offer " \ - "not only detailed descriptions but also provide nuanced explanations, examples, or anecdotes to enrich " \ - "their response. Make sure that the generated question does not contain forbidden subjects in muslim countries." \ - "Provide your response in this json format: {'topic': 'topic','question': 'question'}" - token_count = count_tokens(gen_sp1_question)["n_tokens"] - response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp1_question, token_count, GEN_FIELDS, - GEN_QUESTION_TEMPERATURE) + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + '{"topic": "topic", "question": "question"}') + }, + { + "role": "user", + "content": ( + 'Craft a thought-provoking question of ' + difficulty + ' difficulty for IELTS Speaking Part 1 ' + 'that encourages candidates to delve deeply into ' + 'personal experiences, preferences, or insights on the topic ' + 'of "' + topic + '". Instruct the candidate ' + 'to offer not only detailed ' + 'descriptions but also provide ' + 'nuanced explanations, examples, ' + 'or anecdotes to enrich their response. ' + 'Make sure that the generated question ' + 'does not contain forbidden subjects in ' + 'muslim countries.') + } + ] + token_count = count_total_tokens(messages) + response = make_openai_call(GPT_4_O, messages, token_count, ["topic"], + GEN_QUESTION_TEMPERATURE) response["type"] = 1 response["difficulty"] = difficulty response["topic"] = topic @@ -554,33 +572,53 @@ def grade_speaking_task_2(): logging.info("POST - speaking_task_2 - " + str(request_id) + " - Transcripted answer: " + answer) if has_x_words(answer, 20): - message = ("Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a " - "strict assessment that penalizes errors. Deduct points for deviations from the task, and " - "assign a score of 0 if the response fails to address the question. Additionally, provide " - "detailed commentary highlighting both strengths and weaknesses in the response. Present your " - "evaluation in JSON format with " - "the following structure: {'comment': 'comment about answer quality', 'overall': 0.0, " - "'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range " - "and Accuracy': 0.0, " - "'Pronunciation': 0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'") - token_count = count_tokens(message)["n_tokens"] + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + '{"comment": "comment about answer quality", "overall": 0.0, ' + '"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, ' + '"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}') + }, + { + "role": "user", + "content": ( + 'Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a ' + 'strict assessment that penalizes errors. Deduct points for deviations from the task, and ' + 'assign a score of 0 if the response fails to address the question. Additionally, provide ' + 'detailed commentary highlighting both strengths and weaknesses in the response.' + '\n Question: "' + question + '" \n Answer: "' + answer + '"') + } + ] + token_count = count_total_tokens(messages) logging.info("POST - speaking_task_2 - " + str(request_id) + " - Requesting grading of the answer.") - response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, - ["comment"], + response = make_openai_call(GPT_3_5_TURBO, messages, token_count,["comment"], GRADING_TEMPERATURE) logging.info("POST - speaking_task_2 - " + str(request_id) + " - Answer graded: " + str(response)) - perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following " - "Speaking Part 2 question: '" + question + "'") - token_count = count_tokens(perfect_answer_message)["n_tokens"] + perfect_answer_messages = [ + { + "role": "system", + "content": ('You are a helpful assistant designed to output JSON on this format: ' + '{"answer": "perfect answer"}') + }, + { + "role": "user", + "content": ( + 'Provide a perfect answer according to ielts grading system to the following ' + 'Speaking Part 2 question: "' + question + '"') + } + ] + token_count = count_total_tokens(perfect_answer_messages) logging.info("POST - speaking_task_2 - " + str(request_id) + " - Requesting perfect answer.") - response['perfect_answer'] = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, - perfect_answer_message, - token_count, - None, - GEN_QUESTION_TEMPERATURE) + response['perfect_answer'] = make_openai_call(GPT_3_5_TURBO, + perfect_answer_messages, + token_count, + ["answer"], + GEN_QUESTION_TEMPERATURE)["answer"] logging.info("POST - speaking_task_2 - " + str( request_id) + " - Perfect answer: " + response['perfect_answer']) @@ -622,15 +660,31 @@ def get_speaking_task_2_question(): difficulty = request.args.get("difficulty", default=random.choice(difficulties)) topic = request.args.get("topic", default=random.choice(mti_topics)) try: - gen_sp2_question = "Create a question of " + difficulty + " difficulty for IELTS Speaking Part 2 that encourages candidates to narrate a personal experience " \ - "or story related to the topic of '" + topic + "'. Include 3 prompts that guide the candidate to describe " \ - "specific aspects of the experience, such as details about the situation, their actions, and the " \ - "reasons it left a lasting impression. Make sure that the generated question does not contain forbidden subjects in muslim countries." \ - "Provide your response in this json format: {'topic': 'topic','question': 'question', " \ - "'prompts': ['prompt_1', 'prompt_2', 'prompt_3']}" - token_count = count_tokens(gen_sp2_question)["n_tokens"] - response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp2_question, token_count, GEN_FIELDS, - GEN_QUESTION_TEMPERATURE) + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + '{"topic": "topic", "question": "question"}') + }, + { + "role": "user", + "content": ( + 'Craft a thought-provoking question of ' + difficulty + ' difficulty for IELTS Speaking Part 2 ' + 'that encourages candidates to narrate a ' + 'personal experience or story related to the topic ' + 'of "' + topic + '". Include 3 prompts that ' + 'guide the candidate to describe ' + 'specific aspects of the experience, ' + 'such as details about the situation, ' + 'their actions, and the reasons it left a ' + 'lasting impression. Make sure that the ' + 'generated question does not contain ' + 'forbidden subjects in muslim countries.') + } + ] + token_count = count_total_tokens(messages) + response = make_openai_call(GPT_4_O, messages, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE) response["type"] = 2 response["difficulty"] = difficulty response["topic"] = topic @@ -645,15 +699,25 @@ def get_speaking_task_3_question(): difficulty = request.args.get("difficulty", default=random.choice(difficulties)) topic = request.args.get("topic", default=random.choice(mti_topics)) try: - gen_sp3_question = "Formulate a set of 3 questions of " + difficulty + " difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a " \ - "meaningful discussion on the topic of '" + topic + "'. Provide inquiries, ensuring " \ - "they explore various aspects, perspectives, and implications related to the topic. " \ - "Make sure that the generated question does not contain forbidden subjects in muslim countries." \ - "Provide your response in this json format: {'topic': 'topic','questions': ['question', " \ - "'question', 'question']}" - token_count = count_tokens(gen_sp3_question)["n_tokens"] - response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp3_question, token_count, GEN_FIELDS, - GEN_QUESTION_TEMPERATURE) + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + '{"topic": "topic", "questions": ["question", "question", "question"]}') + }, + { + "role": "user", + "content": ( + 'Formulate a set of 3 questions of ' + difficulty + ' difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a ' + 'meaningful discussion on the topic of "' + topic + '". Provide inquiries, ensuring ' + 'they explore various aspects, perspectives, and implications related to the topic.' + 'Make sure that the generated question does not contain forbidden subjects in muslim countries.') + + } + ] + token_count = count_total_tokens(messages) + response = make_openai_call(GPT_4_O, messages, token_count, GEN_FIELDS, GEN_QUESTION_TEMPERATURE) # Remove the numbers from the questions only if the string starts with a number response["questions"] = [re.sub(r"^\d+\.\s*", "", question) if re.match(r"^\d+\.", question) else question for question in response["questions"]] @@ -706,16 +770,39 @@ def grade_speaking_task_3(): "Pronunciation": 0 } } - perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following " - "Speaking Part 3 question: '" + item["question"] + "'") - token_count = count_tokens(perfect_answer_message)["n_tokens"] + + perfect_answer_messages = [ + { + "role": "system", + "content": ('You are a helpful assistant designed to output JSON on this format: ' + '{"answer": "perfect answer"}') + }, + { + "role": "user", + "content": ( + 'Provide a perfect answer according to ielts grading system to the following ' + 'Speaking Part 3 question: "' + item["question"] + '"') + } + ] + token_count = count_total_tokens(perfect_answer_messages) logging.info("POST - speaking_task_3 - " + str( request_id) + " - Requesting perfect answer for question: " + item["question"]) - perfect_answers.append(make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, - perfect_answer_message, - token_count, - None, - GEN_QUESTION_TEMPERATURE)) + perfect_answers.append(make_openai_call(GPT_3_5_TURBO, + perfect_answer_messages, + token_count, + ["answer"], + GEN_QUESTION_TEMPERATURE)) + + messages = [ + { + "role": "system", + "content": ( + 'You are a helpful assistant designed to output JSON on this format: ' + '{"comment": "comment about answer quality", "overall": 0.0, ' + '"task_response": {"Fluency and Coherence": 0.0, "Lexical Resource": 0.0, ' + '"Grammatical Range and Accuracy": 0.0, "Pronunciation": 0.0}}') + } + ] message = ( "Evaluate the given Speaking Part 3 response based on the IELTS grading system, ensuring a " "strict assessment that penalizes errors. Deduct points for deviations from the task, and " @@ -732,17 +819,16 @@ def grade_speaking_task_3(): request_id) + " - Formatted answers and questions for prompt: " + formatted_text) message += formatted_text - message += ( - "'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', " - "'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, " - "'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}") - token_count = count_tokens(message)["n_tokens"] + messages.append({ + "role": "user", + "content": message + }) + + token_count = count_total_tokens(messages) logging.info("POST - speaking_task_3 - " + str(request_id) + " - Requesting grading of the answers.") - response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, - ["comment"], - GRADING_TEMPERATURE) + response = make_openai_call(GPT_3_5_TURBO, messages, token_count, ["comment"], GRADING_TEMPERATURE) logging.info("POST - speaking_task_3 - " + str(request_id) + " - Answers graded: " + str(response)) logging.info("POST - speaking_task_3 - " + str(request_id) + " - Adding perfect answers to response.")