381 lines
22 KiB
Python
381 lines
22 KiB
Python
from enum import Enum
|
||
|
||
|
||
class QuestionType(Enum):
|
||
LISTENING_SECTION_1 = "Listening Section 1"
|
||
LISTENING_SECTION_2 = "Listening Section 2"
|
||
LISTENING_SECTION_3 = "Listening Section 3"
|
||
LISTENING_SECTION_4 = "Listening Section 4"
|
||
WRITING_TASK_1 = "Writing Task 1"
|
||
WRITING_TASK_2 = "Writing Task 2"
|
||
SPEAKING_1 = "Speaking Task Part 1"
|
||
SPEAKING_2 = "Speaking Task Part 2"
|
||
|
||
|
||
def get_grading_messages(question_type: QuestionType, question: str, answer: str, context: str = None):
|
||
if QuestionType.WRITING_TASK_1 == question_type:
|
||
messages = [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS examiner.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": f"The question you have to grade is of type Writing Task 1 and is the following: {question}",
|
||
}
|
||
]
|
||
|
||
if not (context is None or context == ""):
|
||
messages.append({
|
||
"role": "user",
|
||
"content": f"To grade the previous question, bear in mind the following context: {context}",
|
||
})
|
||
|
||
messages.extend([
|
||
{
|
||
"role": "user",
|
||
"content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
|
||
"with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, "
|
||
"'task_response': {'Task Achievement': 8.0, 'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, "
|
||
"'Grammatical Range and Accuracy': 6.0}}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Example output: { 'comment': 'Overall, the response is good but there are some areas that need "
|
||
"improvement.\n\nIn terms of Task Achievement, the writer has addressed all parts of the question "
|
||
"and has provided a clear opinion on the topic. However, some of the points made are not fully "
|
||
"developed or supported with examples.\n\nIn terms of Coherence and Cohesion, there is a clear "
|
||
"structure to the response with an introduction, body paragraphs and conclusion. However, there "
|
||
"are some issues with cohesion as some sentences do not flow smoothly from one to another.\n\nIn "
|
||
"terms of Lexical Resource, there is a good range of vocabulary used throughout the response and "
|
||
"some less common words have been used effectively.\n\nIn terms of Grammatical Range and Accuracy, "
|
||
"there are some errors in grammar and sentence structure which affect clarity in places.\n\nOverall, "
|
||
"this response would score a band 6.5.', 'overall': 6.5, 'task_response': "
|
||
"{ 'Coherence and Cohesion': 6.5, 'Grammatical Range and Accuracy': 6.0, 'Lexical Resource': 7.0, "
|
||
"'Task Achievement': 7.0}}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": f"Evaluate this answer according to ielts grading system: {answer}",
|
||
},
|
||
])
|
||
return messages
|
||
elif QuestionType.WRITING_TASK_2 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS examiner.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": f"The question you have to grade is of type Writing Task 2 and is the following: {question}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
|
||
"with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, "
|
||
"'task_response': {'Task Achievement': 8.0, 'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, "
|
||
"'Grammatical Range and Accuracy': 6.0}}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Example output: { 'comment': 'Overall, the response is good but there are some areas that need "
|
||
"improvement.\n\nIn terms of Task Achievement, the writer has addressed all parts of the question "
|
||
"and has provided a clear opinion on the topic. However, some of the points made are not fully "
|
||
"developed or supported with examples.\n\nIn terms of Coherence and Cohesion, there is a clear "
|
||
"structure to the response with an introduction, body paragraphs and conclusion. However, there "
|
||
"are some issues with cohesion as some sentences do not flow smoothly from one to another.\n\nIn "
|
||
"terms of Lexical Resource, there is a good range of vocabulary used throughout the response and "
|
||
"some less common words have been used effectively.\n\nIn terms of Grammatical Range and Accuracy, "
|
||
"there are some errors in grammar and sentence structure which affect clarity in places.\n\nOverall, "
|
||
"this response would score a band 6.5.', 'overall': 6.5, 'task_response': "
|
||
"{ 'Coherence and Cohesion': 6.5, 'Grammatical Range and Accuracy': 6.0, 'Lexical Resource': 7.0, "
|
||
"'Task Achievement': 7.0}}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": f"Evaluate this answer according to ielts grading system: {answer}",
|
||
},
|
||
]
|
||
elif QuestionType.SPEAKING_1 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS examiner.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": f"The question you have to grade is of type Speaking Task Part 1 and is the following: {question}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
|
||
"with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, "
|
||
"'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, "
|
||
"'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has "
|
||
"given examples of how they spend their weekends. However, there are some issues with grammar and "
|
||
"pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks "
|
||
"clearly and smoothly with only minor hesitations. They have also provided a well-organized response "
|
||
"that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary "
|
||
"related to weekend activities but there are some errors in word choice that affect the meaning of "
|
||
"their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple "
|
||
"and complex sentence structures but there are some errors in subject-verb agreement and preposition "
|
||
"use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some "
|
||
"issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, "
|
||
"'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0,"
|
||
" 'Pronunciation': 6.0}}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": f"Evaluate this answer according to ielts grading system: {answer}",
|
||
},
|
||
]
|
||
elif QuestionType.SPEAKING_2 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS examiner.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": f"The question you have to grade is of type Speaking Task Part 2 and is the following: {question}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
|
||
"with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, "
|
||
"'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, "
|
||
"'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has "
|
||
"given examples of how they spend their weekends. However, there are some issues with grammar and "
|
||
"pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks "
|
||
"clearly and smoothly with only minor hesitations. They have also provided a well-organized response "
|
||
"that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary "
|
||
"related to weekend activities but there are some errors in word choice that affect the meaning of "
|
||
"their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple "
|
||
"and complex sentence structures but there are some errors in subject-verb agreement and preposition "
|
||
"use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some "
|
||
"issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, "
|
||
"'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0,"
|
||
" 'Pronunciation': 6.0}}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": f"Evaluate this answer according to ielts grading system: {answer}",
|
||
},
|
||
]
|
||
else:
|
||
raise Exception("Question type not implemented: " + question_type.value)
|
||
|
||
|
||
def get_question_gen_messages(question_type: QuestionType):
|
||
if QuestionType.LISTENING_SECTION_1 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS program that generates questions for the exams.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Provide me with a transcript similar to the ones in ielts exam Listening Section 1. "
|
||
"Create an engaging transcript simulating a conversation related to a unique type of service "
|
||
"that requires getting the customer's details. Make sure to include specific details "
|
||
"and descriptions to bring"
|
||
"the scenario to life. After the transcript, please "
|
||
"generate a 'form like' fill in the blanks exercise with 6 form fields (ex: name, date of birth)"
|
||
" to fill related to the customer's details. Finally, "
|
||
"provide the answers for the exercise. The response must be a json following this format: "
|
||
"{ 'type': '<type of registration (ex: hotel, gym, english course, etc)>', "
|
||
"'transcript': '<transcript of just the conversation about a registration of some sort, "
|
||
"identify the person talking in each speech line>', "
|
||
"'exercise': { 'form field': { '1': '<form field 1>', '2': '<form field 2>', "
|
||
"'3': '<form field 3>', '4': '<form field 4>', "
|
||
"'5': '<form field 5>', '6': '<form field 5>' }, "
|
||
"'answers': {'1': '<answer to fill blank space in form field 1>', '2': '<answer to fill blank "
|
||
"space in form field 2>', '3': '<answer to fill blank space in form field 3>', "
|
||
"'4': '<answer to fill blank space in form field 4>', '5': '<answer to fill blank space in form field 5>',"
|
||
" '6': '<answer to fill blank space in form field 6>'}}}",
|
||
}
|
||
]
|
||
elif QuestionType.LISTENING_SECTION_2 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS program that generates questions for the exams.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Provide me with a transcript similar to the ones in ielts exam Listening section 2. After the transcript, please "
|
||
"generate a fill in the blanks exercise with 6 statements related to the text content. Finally, "
|
||
"provide the answers for the exercise. The response must be a json following this format: "
|
||
"{ 'transcript': 'transcript about some subject', 'exercise': { 'statements': { '1': 'statement 1 "
|
||
"with a blank space to fill', '2': 'statement 2 with a blank space to fill', '3': 'statement 3 with a "
|
||
"blank space to fill', '4': 'statement 4 with a blank space to fill', '5': 'statement 5 with a blank "
|
||
"space to fill', '6': 'statement 6 with a blank space to fill' }, "
|
||
"'answers': {'1': 'answer to fill blank space in statement 1', '2': 'answer to fill blank "
|
||
"space in statement 2', '3': 'answer to fill blank space in statement 3', "
|
||
"'4': 'answer to fill blank space in statement 4', '5': 'answer to fill blank space in statement 5',"
|
||
" '6': 'answer to fill blank space in statement 6'}}}",
|
||
}
|
||
]
|
||
elif QuestionType.LISTENING_SECTION_3 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS program that generates questions for the exams.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Provide me with a transcript similar to the ones in ielts exam Listening section 3. After the transcript, please "
|
||
"generate 4 multiple choice questions related to the text content. Finally, "
|
||
"provide the answers for the exercise. The response must be a json following this format: "
|
||
"{ 'transcript': 'generated transcript similar to the ones in ielts exam Listening section 3', "
|
||
"'exercise': { 'questions': [ { 'question': "
|
||
"'question 1', 'options': ['option 1', 'option 2', 'option 3', 'option 4'], 'answer': 1}, "
|
||
"{'question': 'question 2', 'options': ['option 1', 'option 2', 'option 3', 'option 4'], "
|
||
"'answer': 3}, {'question': 'question 3', 'options': ['option 1', 'option 2', 'option 3', "
|
||
"'option 4'], 'answer': 0}, {'question': 'question 4', 'options': ['option 1', 'option 2', "
|
||
"'option 3', 'option 4'], 'answer': 2}]}}",
|
||
}
|
||
]
|
||
elif QuestionType.LISTENING_SECTION_4 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS program that generates questions for the exams.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Provide me with a transcript similar to the ones in ielts exam Listening section 4. After the transcript, please "
|
||
"generate 4 completion-type questions related to the text content to complete with 1 word. Finally, "
|
||
"provide the answers for the exercise. The response must be a json following this format: "
|
||
"{ 'transcript': 'generated transcript similar to the ones in ielts exam Listening section 4', "
|
||
"'exercise': [ { 'question': 'question 1', 'answer': 'answer 1'}, "
|
||
"{'question': 'question 2', 'answer': 'answer 2'}, {'question': 'question 3', 'answer': 'answer 3'}, "
|
||
"{'question': 'question 4', 'answer': 'answer 4'}]}",
|
||
}
|
||
]
|
||
elif QuestionType.WRITING_TASK_2 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS program that generates questions for the exams.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "The question you have to generate is of type Writing Task 2.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "It is mandatory for you to provide your response with the question "
|
||
"just with the following json format: {'question': 'question'}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Example output: { 'question': 'We are becoming increasingly dependent on computers. "
|
||
"They are used in businesses, hospitals, crime detection and even to fly planes. What things will "
|
||
"they be used for in the future? Is this dependence on computers a good thing or should we he more "
|
||
"auspicious of their benefits?'}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Generate a question for IELTS exam Writing Task 2.",
|
||
},
|
||
]
|
||
elif QuestionType.SPEAKING_1 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS program that generates questions for the exams.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "The question you have to generate is of type Speaking Task Part 1.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "It is mandatory for you to provide your response with the question "
|
||
"just with the following json format: {'question': 'question'}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Example output: { 'question': 'Let’s talk about your home town or village. "
|
||
"What kind of place is it? What’s the most interesting part of your town/village? "
|
||
"What kind of jobs do the people in your town/village do? "
|
||
"Would you say it’s a good place to live? (Why?)'}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Generate a question for IELTS exam Speaking Task.",
|
||
},
|
||
]
|
||
elif QuestionType.SPEAKING_2 == question_type:
|
||
return [
|
||
{
|
||
"role": "user",
|
||
"content": "You are a IELTS program that generates questions for the exams.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "The question you have to generate is of type Speaking Task Part 2.",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "It is mandatory for you to provide your response with the question "
|
||
"just with the following json format: {'question': 'question'}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Example output: { 'question': 'Describe something you own which is very important to you. "
|
||
"You should say: where you got it from how long you have had it what you use it for and "
|
||
"explain why it is important to you.'}",
|
||
},
|
||
{
|
||
"role": "user",
|
||
"content": "Generate a question for IELTS exam Speaking Task.",
|
||
},
|
||
]
|
||
else:
|
||
raise Exception("Question type not implemented: " + question_type.value)
|
||
|
||
|
||
def get_question_tips(question: str, answer: str, correct_answer: str, context: str = None):
|
||
messages = [
|
||
{
|
||
"role": "system",
|
||
"content": "You are a IELTS exam program that analyzes incorrect answers to questions and gives tips to "
|
||
"help students understand why it was a wrong answer and explains how their thought process "
|
||
"should have been. The tip should refer to the context and question.",
|
||
}
|
||
]
|
||
|
||
if not (context is None or context == ""):
|
||
messages.append({
|
||
"role": "system",
|
||
"content": f"This is the context for the question: {context}",
|
||
})
|
||
|
||
messages.extend([
|
||
{
|
||
"role": "system",
|
||
"content": f"This is the question: {question}",
|
||
},
|
||
{
|
||
"role": "system",
|
||
"content": f"This is the answer: {answer}",
|
||
},
|
||
{
|
||
"role": "system",
|
||
"content": f"This is the correct answer: {correct_answer}",
|
||
}
|
||
])
|
||
|
||
return messages
|