devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix level exam generation

Browse Source
This commit is contained in:
Cristiano Ferreira
2024-02-17 15:40:19 +00:00
parent a200b29dba
commit 1bd012d340
3 changed files with 84 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
helper/constants.py
View File

@@ -10,6 +10,7 @@ GEN_QUESTION_TEMPERATURE = 0.7
GPT_3_5_TURBO = "gpt-3.5-turbo" GPT_3_5_TURBO = "gpt-3.5-turbo"
GPT_3_5_TURBO_16K = "gpt-3.5-turbo-16k" GPT_3_5_TURBO_16K = "gpt-3.5-turbo-16k"
GPT_3_5_TURBO_INSTRUCT = "gpt-3.5-turbo-instruct" GPT_3_5_TURBO_INSTRUCT = "gpt-3.5-turbo-instruct"
GPT_4_PREVIEW = "gpt-4-turbo-preview"
GRADING_FIELDS = ['comment', 'overall', 'task_response'] GRADING_FIELDS = ['comment', 'overall', 'task_response']
GEN_FIELDS = ['topic'] GEN_FIELDS = ['topic']

85
helper/exercises.py
View File

@@ -6,7 +6,7 @@ import uuid
from helper.api_messages import QuestionType from helper.api_messages import QuestionType
from helper.firebase_helper import get_all from helper.firebase_helper import get_all
from helper.openai_interface import make_openai_instruct_call from helper.openai_interface import make_openai_instruct_call, make_openai_call
from helper.token_counter import count_tokens from helper.token_counter import count_tokens
from helper.constants import * from helper.constants import *
from wonderwords import RandomWord from wonderwords import RandomWord
@@ -682,47 +682,58 @@ def gen_multiple_choice_level(quantity: int, start_id=1):
"questions and some advanced questions. Ensure that the questions cover a range of topics such as " \ "questions and some advanced questions. Ensure that the questions cover a range of topics such as " \
"verb tense, subject-verb agreement, pronoun usage, sentence structure, and punctuation. Make sure " \ "verb tense, subject-verb agreement, pronoun usage, sentence structure, and punctuation. Make sure " \
"every question only has 1 correct answer." "every question only has 1 correct answer."
messages = [{
"role": "user",
"content": gen_multiple_choice_for_text
}]
token_count = count_tokens(gen_multiple_choice_for_text)["n_tokens"] - 300 token_count = count_tokens(gen_multiple_choice_for_text)["n_tokens"] - 300
mc_questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_multiple_choice_for_text, token_count, mc_questions = make_openai_call(GPT_4_PREVIEW, messages, token_count,
None, None,
GEN_QUESTION_TEMPERATURE) GEN_QUESTION_TEMPERATURE)
split_mc_questions = mc_questions.split('13') if not '25' in mc_questions:
return gen_multiple_choice_level(quantity, start_id)
else:
split_mc_questions = mc_questions.split('13')
parse_mc_questions = ('Parse the questions into this json format: {"questions": [{"id": "9", "options": ' parse_mc_questions = ('Parse the questions into this json format: \n\'{"questions": [{"id": "9", "options": '
'[{"id": "A", "text": ' '[{"id": "A", "text": '
'"And"}, {"id": "B", "text": "Cat"}, {"id": "C", "text": ' '"And"}, {"id": "B", "text": "Cat"}, {"id": "C", "text": '
'"Happy"}, {"id": "D", "text": "Jump"}], ' '"Happy"}, {"id": "D", "text": "Jump"}], '
'"prompt": "Which of the following is a conjunction?", ' '"prompt": "Which of the following is a conjunction?", '
'"solution": "A", "variant": "text"}]}. ' '"solution": "A", "variant": "text"}]}\'\n '
'\nThe questions: "' + split_mc_questions[0] + '"') '\nThe questions: "' + split_mc_questions[0] + '"')
token_count = count_tokens(parse_mc_questions, model_name=GPT_3_5_TURBO_INSTRUCT)["n_tokens"] token_count = count_tokens(parse_mc_questions, model_name=GPT_3_5_TURBO_INSTRUCT)["n_tokens"]
question = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count, question = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count,
["questions"], ["questions"],
GEN_QUESTION_TEMPERATURE) GEN_QUESTION_TEMPERATURE)
parse_mc_questions = ('Parse the questions into this json format: {"questions": [{"id": "9", "options": ' print(question)
'[{"id": "A", "text": ' parse_mc_questions = ('Parse the questions into this json format: \n\'{"questions": [{"id": "9", "options": '
'"And"}, {"id": "B", "text": "Cat"}, {"id": "C", "text": ' '[{"id": "A", "text": '
'"Happy"}, {"id": "D", "text": "Jump"}], ' '"And"}, {"id": "B", "text": "Cat"}, {"id": "C", "text": '
'"prompt": "Which of the following is a conjunction?", ' '"Happy"}, {"id": "D", "text": "Jump"}], '
'"solution": "A", "variant": "text"}]}. ' '"prompt": "Which of the following is a conjunction?", '
'\nThe questions: "' + '13' + split_mc_questions[1] + '"') '"solution": "A", "variant": "text"}]}\'\n '
token_count = count_tokens(parse_mc_questions, model_name=GPT_3_5_TURBO_INSTRUCT)["n_tokens"] '\nThe questions: "' + '13' + split_mc_questions[1] + '"')
question_2 = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count, token_count = count_tokens(parse_mc_questions, model_name=GPT_3_5_TURBO_INSTRUCT)["n_tokens"]
["questions"], question_2 = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count,
GEN_QUESTION_TEMPERATURE) ["questions"],
question["questions"].extend(question_2["questions"]) GEN_QUESTION_TEMPERATURE)
print(question_2)
question["questions"].extend(question_2["questions"])
all_exams = get_all("level") all_exams = get_all("level")
seen_keys = set() seen_keys = set()
for i in range(len(question["questions"])): for i in range(len(question["questions"])):
question["questions"][i], seen_keys = replace_exercise_if_exists(all_exams, question["questions"][i], question, question["questions"][i], seen_keys = replace_exercise_if_exists(all_exams, question["questions"][i], question,
seen_keys) seen_keys)
return { return {
"id": str(uuid.uuid4()), "id": str(uuid.uuid4()),
"prompt": "Select the appropriate option.", "prompt": "Select the appropriate option.",
"questions": fix_exercise_ids(question, start_id)["questions"], "questions": fix_exercise_ids(question, start_id)["questions"],
"type": "multipleChoice", "type": "multipleChoice",
} }
def replace_exercise_if_exists(all_exams, current_exercise, current_exam, seen_keys): def replace_exercise_if_exists(all_exams, current_exercise, current_exam, seen_keys):

45
helper/openai_interface.py
View File

@@ -140,19 +140,25 @@ def make_openai_call(model, messages, token_count, fields_to_check, temperature)
top_p=float(TOP_P), top_p=float(TOP_P),
frequency_penalty=float(FREQUENCY_PENALTY), frequency_penalty=float(FREQUENCY_PENALTY),
messages=messages messages=messages
) )["choices"][0]["message"]["content"]
if has_blacklisted_words(result) and try_count < TRY_LIMIT:
try_count = try_count + 1
return make_openai_call(model, messages, token_count, fields_to_check, temperature)
elif has_blacklisted_words(result) and try_count >= TRY_LIMIT:
return ""
if fields_to_check is None: if fields_to_check is None:
return result["choices"][0]["message"]["content"] return result.replace("\n\n", " ").strip()
processed_response = process_response(result["choices"][0]["message"]["content"], fields_to_check[0]) processed_response = process_response(result, fields_to_check[0])
if check_fields(processed_response, fields_to_check) is False and try_count < TRY_LIMIT: if check_fields(processed_response, fields_to_check) is False and try_count < TRY_LIMIT:
try_count = try_count + 1 try_count = try_count + 1
return make_openai_call(model, messages, token_count, fields_to_check, temperature) return make_openai_call(model, messages, token_count, fields_to_check, temperature)
elif try_count >= TRY_LIMIT: elif try_count >= TRY_LIMIT:
try_count = 0 try_count = 0
return result["choices"][0]["message"]["content"] return result
else: else:
try_count = 0 try_count = 0
return processed_response return processed_response
@@ -176,13 +182,22 @@ def make_openai_instruct_call(model, message: str, token_count, fields_to_check,
if fields_to_check is None: if fields_to_check is None:
return response.replace("\n\n", " ").strip() return response.replace("\n\n", " ").strip()
processed_response = process_response(response, fields_to_check[0]) response = remove_special_characters_from_beginning(response)
if check_fields(processed_response, fields_to_check) is False and try_count < TRY_LIMIT: if response[0] != "{" and response[0] != '"':
try_count = try_count + 1 response = "{\"" + response
if not response.endswith("}"):
response = response + "}"
try:
processed_response = process_response(response, fields_to_check[0])
if check_fields(processed_response, fields_to_check) is False and try_count < TRY_LIMIT:
try_count = try_count + 1
return make_openai_instruct_call(model, message, token_count, fields_to_check, temperature)
else:
try_count = 0
return processed_response
except Exception as e:
return make_openai_instruct_call(model, message, token_count, fields_to_check, temperature) return make_openai_instruct_call(model, message, token_count, fields_to_check, temperature)
else:
try_count = 0
return processed_response
# GRADING SUMMARY # GRADING SUMMARY
@@ -275,3 +290,13 @@ def get_speaking_corrections(text):
def has_blacklisted_words(text: str): def has_blacklisted_words(text: str):
text_lower = text.lower() text_lower = text.lower()
return any(word in text_lower for word in BLACKLISTED_WORDS) return any(word in text_lower for word in BLACKLISTED_WORDS)
def remove_special_characters_from_beginning(string):
cleaned_string = string.lstrip('\n')
if string.startswith("'") or string.startswith('"'):
cleaned_string = string[1:]
if cleaned_string.endswith('"'):
return cleaned_string[:-1]
else:
return cleaned_string