Add a few more black listed words.

This reverts commit 1c888f22e2.

.env

@@ -1,3 +1,5 @@
 OPENAI_API_KEY=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
 JWT_SECRET_KEY=6e9c124ba92e8814719dcb0f21200c8aa4d0f119a994ac5e06eb90a366c83ab2
 JWT_TEST_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0In0.Emrs2D3BmMP4b3zMjw0fJTPeyMwWEBDbxx2vvaWguO0
+GOOGLE_APPLICATION_CREDENTIALS=firebase-configs/storied-phalanx-349916.json
+HEY_GEN_TOKEN=MjY4MDE0MjdjZmNhNDFmYTlhZGRkNmI3MGFlMzYwZDItMTY5NTExNzY3MA==

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 .idea
+.env

.idea/ielts-be.iml

@@ -10,6 +10,9 @@
     <orderEntry type="jdk" jdkName="Python 3.9" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
+  <component name="PackageRequirementsSettings">
+    <option name="versionSpecifier" value="Don't specify version" />
+  </component>
   <component name="TemplatesService">
     <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
     <option name="TEMPLATE_FOLDERS">

Dockerfile

@@ -11,6 +11,8 @@ ENV APP_HOME /app
 WORKDIR $APP_HOME
 COPY . ./
 
+RUN apt update && apt install -y ffmpeg
+
 # Install production dependencies.
 RUN pip install --no-cache-dir -r requirements.txt
 

app.py

@@ -1,12 +1,26 @@
+import threading
+
 from flask import Flask, request
 from flask_jwt_extended import JWTManager, jwt_required
 from functools import reduce
-from helper.token_counter import count_tokens
-from helper.openai_interface import make_openai_call
+
+from helper.ExamVariant import ExamVariant
+from helper.api_messages import *
+from helper.exercises import *
+from helper.file_helper import delete_files_older_than_one_day
+from helper.firebase_helper import *
+from helper.heygen_api import create_video, create_videos_and_save_to_db
+from heygen.AvatarEnum import AvatarEnum
+from helper.speech_to_text_helper import *
+from helper.openai_interface import *
 import os
+import re
+import logging
 
 from dotenv import load_dotenv
 
+from templates.question_templates import *
+
 load_dotenv()
 
 app = Flask(__name__)
@@ -14,91 +28,818 @@ app = Flask(__name__)
 app.config['JWT_SECRET_KEY'] = os.getenv("JWT_SECRET_KEY")
 jwt = JWTManager(app)
 
-GRADING_TEMPERATURE = 0.1
-GEN_QUESTION_TEMPERATURE = 0.7
-WRITING_TASK_2_POST_FIELDS = ['overall', 'comment', 'task_response']
-WRITING_TASK_2_GET_FIELDS = ['question']
+# Initialize Firebase Admin SDK
+cred = credentials.Certificate(os.getenv("GOOGLE_APPLICATION_CREDENTIALS"))
+firebase_admin.initialize_app(cred)
+
+thread_event = threading.Event()
+
+# Configure logging
+logging.basicConfig(level=logging.DEBUG,  # Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+                    format='%(asctime)s - %(levelname)s - %(message)s')
+
+
+@app.route('/healthcheck', methods=['GET'])
+def healthcheck():
+    return {"healthy": True}
+
+
+@app.route('/listening_section_1', methods=['GET'])
+@jwt_required()
+def get_listening_section_1_question():
+    try:
+        delete_files_older_than_one_day(AUDIO_FILES_PATH)
+        # Extract parameters from the URL query string
+        topic = request.args.get('topic', default=random.choice(two_people_scenarios))
+        req_exercises = request.args.getlist('exercises')
+        difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+
+        if (len(req_exercises) == 0):
+            req_exercises = random.sample(LISTENING_EXERCISE_TYPES, 1)
+
+        number_of_exercises_q = divide_number_into_parts(TOTAL_LISTENING_SECTION_1_EXERCISES, len(req_exercises))
+
+        unprocessed_conversation, processed_conversation = generate_listening_1_conversation(topic)
+
+        app.logger.info("Generated conversation: " + str(processed_conversation))
+
+        start_id = 1
+        exercises = generate_listening_conversation_exercises(unprocessed_conversation, req_exercises,
+                                                              number_of_exercises_q,
+                                                              start_id, difficulty)
+        return {
+            "exercises": exercises,
+            "text": processed_conversation,
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/listening_section_2', methods=['GET'])
+@jwt_required()
+def get_listening_section_2_question():
+    try:
+        delete_files_older_than_one_day(AUDIO_FILES_PATH)
+        # Extract parameters from the URL query string
+        topic = request.args.get('topic', default=random.choice(social_monologue_contexts))
+        req_exercises = request.args.getlist('exercises')
+        difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+
+        if (len(req_exercises) == 0):
+            req_exercises = random.sample(LISTENING_EXERCISE_TYPES, 2)
+
+        number_of_exercises_q = divide_number_into_parts(TOTAL_LISTENING_SECTION_2_EXERCISES, len(req_exercises))
+
+        monologue = generate_listening_2_monologue(topic)
+
+        app.logger.info("Generated monologue: " + str(monologue))
+        start_id = 11
+        exercises = generate_listening_monologue_exercises(monologue, req_exercises, number_of_exercises_q, start_id, difficulty)
+        return {
+            "exercises": exercises,
+            "text": monologue,
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/listening_section_3', methods=['GET'])
+@jwt_required()
+def get_listening_section_3_question():
+    try:
+        delete_files_older_than_one_day(AUDIO_FILES_PATH)
+        # Extract parameters from the URL query string
+        topic = request.args.get('topic', default=random.choice(four_people_scenarios))
+        req_exercises = request.args.getlist('exercises')
+        difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+
+        if (len(req_exercises) == 0):
+            req_exercises = random.sample(LISTENING_EXERCISE_TYPES, 1)
+
+        number_of_exercises_q = divide_number_into_parts(TOTAL_LISTENING_SECTION_3_EXERCISES, len(req_exercises))
+
+        unprocessed_conversation, processed_conversation = generate_listening_3_conversation(topic)
+
+        app.logger.info("Generated conversation: " + str(processed_conversation))
+
+        start_id = 21
+        exercises = generate_listening_conversation_exercises(unprocessed_conversation, req_exercises,
+                                                              number_of_exercises_q,
+                                                              start_id, difficulty)
+        return {
+            "exercises": exercises,
+            "text": processed_conversation,
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/listening_section_4', methods=['GET'])
+@jwt_required()
+def get_listening_section_4_question():
+    try:
+        delete_files_older_than_one_day(AUDIO_FILES_PATH)
+        # Extract parameters from the URL query string
+        topic = request.args.get('topic', default=random.choice(academic_subjects))
+        req_exercises = request.args.getlist('exercises')
+        difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+
+        if (len(req_exercises) == 0):
+            req_exercises = random.sample(LISTENING_EXERCISE_TYPES, 2)
+
+        number_of_exercises_q = divide_number_into_parts(TOTAL_LISTENING_SECTION_4_EXERCISES, len(req_exercises))
+
+        monologue = generate_listening_4_monologue(topic)
+
+        app.logger.info("Generated monologue: " + str(monologue))
+        start_id = 31
+        exercises = generate_listening_monologue_exercises(monologue, req_exercises, number_of_exercises_q, start_id, difficulty)
+        return {
+            "exercises": exercises,
+            "text": monologue,
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/listening', methods=['POST'])
+@jwt_required()
+def save_listening():
+    try:
+        data = request.get_json()
+        parts = data.get('parts')
+        minTimer = data.get('minTimer', LISTENING_MIN_TIMER_DEFAULT)
+        difficulty = data.get('difficulty', random.choice(difficulties))
+        template = getListeningTemplate()
+        template['difficulty'] = difficulty
+        id = str(uuid.uuid4())
+        for i, part in enumerate(parts, start=0):
+            part_template = getListeningPartTemplate()
+
+            file_name = str(uuid.uuid4()) + ".mp3"
+            sound_file_path = AUDIO_FILES_PATH + file_name
+            firebase_file_path = FIREBASE_LISTENING_AUDIO_FILES_PATH + file_name
+            if "conversation" in part["text"]:
+                conversation_text_to_speech(part["text"]["conversation"], sound_file_path)
+            else:
+                text_to_speech(part["text"], sound_file_path)
+            file_url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+
+            part_template["audio"]["source"] = file_url
+            part_template["exercises"] = part["exercises"]
+
+            template['parts'].append(part_template)
+
+        if minTimer != LISTENING_MIN_TIMER_DEFAULT:
+            template["minTimer"] = minTimer
+            template["variant"] = ExamVariant.PARTIAL.value
+        else:
+            template["variant"] = ExamVariant.FULL.value
+
+        (result, id) = save_to_db_with_id("listening", template, id)
+        if result:
+            return {**template, "id": id}
+        else:
+            raise Exception("Failed to save question: " + parts)
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/writing_task1', methods=['POST'])
+@jwt_required()
+def grade_writing_task_1():
+    try:
+        data = request.get_json()
+        question = data.get('question')
+        answer = data.get('answer')
+        if has_words(answer):
+            message = ("Evaluate the given Writing Task 1 response based on the IELTS grading system, ensuring a "
+                       "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
+                       "assign a score of 0 if the response fails to address the question. Additionally, provide an "
+                       "exemplary answer with a minimum of 150 words, along with a detailed commentary highlighting "
+                       "both strengths and weaknesses in the response. Present your evaluation in JSON format with "
+                       "the following structure: {'perfect_answer': 'example perfect answer', 'comment': "
+                       "'comment about answer quality', 'overall': 0.0, 'task_response': {'Task Achievement': 0.0, "
+                       "'Coherence and Cohesion': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range and Accuracy': "
+                       "0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
+            token_count = count_tokens(message)["n_tokens"]
+            response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
+                                                 ["comment"],
+                                                 GEN_QUESTION_TEMPERATURE)
+            response['fixed_text'] = get_fixed_text(answer)
+            return response
+        else:
+            return {
+                'comment': "The answer does not contain any english words.",
+                'overall': 0,
+                'task_response': {
+                    'Coherence and Cohesion': 0,
+                    'Grammatical Range and Accuracy': 0,
+                    'Lexical Resource': 0,
+                    'Task Achievement': 0
+                }
+            }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/writing_task1_general', methods=['GET'])
+@jwt_required()
+def get_writing_task_1_general_question():
+    difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+    try:
+        gen_wt1_question = "Craft a prompt for an IELTS Writing Task 1 General Training exercise that instructs the " \
+                           "student to compose a letter. The prompt should present a specific scenario or situation, " \
+                           "requiring the student to provide information, advice, or instructions within the letter. " \
+                           "Make sure that the generated prompt is of " + difficulty + " difficulty and does not contain forbidden subjects in muslim countries."
+        token_count = count_tokens(gen_wt1_question)["n_tokens"]
+        response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_wt1_question, token_count, None,
+                                             GEN_QUESTION_TEMPERATURE)
+        return {
+            "question": response.strip(),
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
 
 @app.route('/writing_task2', methods=['POST'])
 @jwt_required()
-def grade_writing_task():
-    data = request.get_json()  # Assuming the request data is in JSON format
-    question = data.get('question')
-    answer = data.get('answer')
-    messages = [
-        {
-            "role": "system",
-            "content": "You are a IELTS examiner.",
-        },
-        {
-            "role": "system",
-            "content": f"The question you have to grade is of type Writing Task 2 and is the following: {question}",
-        },
-        {
-            "role": "user",
-            "content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
-                       "in the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, 'task_response': {'Task Achievement': 8.0, "
-                       "'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, 'Grammatical Range and Accuracy': "
-                       "6.0}}",
-        },
-        {
-            "role": "user",
-            "content": "Example output: { 'comment': 'Overall, the response is good but there are some areas that need "
-                       "improvement.\n\nIn terms of Task Achievement, the writer has addressed all parts of the question "
-                       "and has provided a clear opinion on the topic. However, some of the points made are not fully "
-                       "developed or supported with examples.\n\nIn terms of Coherence and Cohesion, there is a clear "
-                       "structure to the response with an introduction, body paragraphs and conclusion. However, there "
-                       "are some issues with cohesion as some sentences do not flow smoothly from one to another.\n\nIn "
-                       "terms of Lexical Resource, there is a good range of vocabulary used throughout the response and "
-                       "some less common words have been used effectively.\n\nIn terms of Grammatical Range and Accuracy, "
-                       "there are some errors in grammar and sentence structure which affect clarity in places.\n\nOverall, "
-                       "this response would score a band 6.5.', 'overall': 6.5, 'task_response': "
-                       "{ 'Coherence and Cohesion': 6.5, 'Grammatical Range and Accuracy': 6.0, 'Lexical Resource': 7.0, "
-                       "'Task Achievement': 7.0}}",
-        },
-        {
-            "role": "user",
-            "content": f"Evaluate this answer according to ielts grading system: {answer}",
-        },
-    ]
-    token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
-                         map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
-    response = make_openai_call(messages, token_count, WRITING_TASK_2_POST_FIELDS, GRADING_TEMPERATURE)
-    return response
+def grade_writing_task_2():
+    try:
+        data = request.get_json()
+        question = data.get('question')
+        answer = data.get('answer')
+        if has_words(answer):
+            message = ("Evaluate the given Writing Task 2 response based on the IELTS grading system, ensuring a "
+                       "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
+                       "assign a score of 0 if the response fails to address the question. Additionally, provide an "
+                       "exemplary answer with a minimum of 250 words, along with a detailed commentary highlighting "
+                       "both strengths and weaknesses in the response. Present your evaluation in JSON format with "
+                       "the following structure: {'perfect_answer': 'example perfect answer', 'comment': "
+                       "'comment about answer quality', 'overall': 0.0, 'task_response': {'Task Achievement': 0.0, "
+                       "'Coherence and Cohesion': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range and Accuracy': "
+                       "0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
+            token_count = count_tokens(message)["n_tokens"]
+            response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
+                                                 ["comment"],
+                                                 GEN_QUESTION_TEMPERATURE)
+            response['fixed_text'] = get_fixed_text(answer)
+            return response
+        else:
+            return {
+                'comment': "The answer does not contain any english words.",
+                'overall': 0,
+                'task_response': {
+                    'Coherence and Cohesion': 0,
+                    'Grammatical Range and Accuracy': 0,
+                    'Lexical Resource': 0,
+                    'Task Achievement': 0
+                }
+            }
+    except Exception as e:
+        return str(e)
 
-@app.route('/writing_task2', methods=['GET'])
+
+@app.route('/writing_task2_general', methods=['GET'])
 @jwt_required()
-def get_writing_task_question():
-    messages = [
-        {
-            "role": "system",
-            "content": "You are a IELTS program that generates questions for the exams.",
-        },
-        {
-            "role": "system",
-            "content": "The question you have to generate is of type Writing Task 2 and is the following.",
-        },
-        {
-            "role": "user",
-            "content": "It is mandatory for you to provide your response with the question "
-                       "in the following json format: {'question': 'question'}",
-        },
-        {
-            "role": "user",
-            "content": "Example output: { 'question': 'We are becoming increasingly dependent on computers. "
-                       "They are used in businesses, hospitals, crime detection and even to fly planes. What things will "
-                       "they be used for in the future? Is this dependence on computers a good thing or should we he more "
-                       "auspicious of their benefits?'}",
-        },
-        {
-            "role": "user",
-            "content": "Generate a question for IELTS exam Writing Task 2.",
-        },
-    ]
-    token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
-                         map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
-    response = make_openai_call(messages, token_count, WRITING_TASK_2_GET_FIELDS, GEN_QUESTION_TEMPERATURE)
-    return response
+def get_writing_task_2_general_question():
+    difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+    try:
+        gen_wt2_question = "Craft a comprehensive question of " + difficulty + " difficulty for IELTS Writing Task 2 General Training that directs the candidate " \
+                           "to delve into an in-depth analysis of contrasting perspectives on a specific topic. The candidate " \
+                           "should be asked to discuss the strengths and weaknesses of both viewpoints, provide evidence or " \
+                           "examples, and present a well-rounded argument before concluding with their personal opinion on the " \
+                           "subject."
+        token_count = count_tokens(gen_wt2_question)["n_tokens"]
+        response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_wt2_question, token_count, None,
+                                             GEN_QUESTION_TEMPERATURE)
+        return {
+            "question": response.strip(),
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
+
+# THE SAVING OF WRITING IS DONE WITHOUT THE API ON THE FRONTEND
+# @app.route('/writing', methods=['POST'])
+# @jwt_required()
+# def save_writing_task():
+#     try:
+#         data = request.get_json()
+#         exercises = data.get('exercises')
+#         template = getWritingTemplate()
+#         id = str(uuid.uuid4())
+#         for i, exercise in enumerate(exercises, start=0):
+#             template["exercises"][i]["prompt"] = exercise
+#
+#         (result, id) = save_to_db_with_id("writing", template, id)
+#         if result:
+#             return {**template, "id": id}
+#         else:
+#             raise Exception("Failed to save writing: " + template)
+#     except Exception as e:
+#         return str(e)
+
+
+@app.route('/speaking_task_1', methods=['POST'])
+@jwt_required()
+def grade_speaking_task_1():
+    delete_files_older_than_one_day(AUDIO_FILES_PATH)
+    sound_file_name = AUDIO_FILES_PATH + str(uuid.uuid4())
+    try:
+        data = request.get_json()
+        question = data.get('question')
+        answer_firebase_path = data.get('answer')
+
+        download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
+        answer = speech_to_text(sound_file_name)
+        if has_10_words(answer):
+            message = ("Evaluate the given Speaking Part 1 response based on the IELTS grading system, ensuring a "
+                       "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
+                       "assign a score of 0 if the response fails to address the question. Additionally, provide "
+                       "detailed commentary highlighting both strengths and weaknesses in the response. Present your "
+                       "evaluation in JSON format with "
+                       "the following structure: {'comment': 'comment about answer quality', 'overall': 0.0, "
+                       "'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range "
+                       "and Accuracy': 0.0, 'Pronunciation': 0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
+            token_count = count_tokens(message)["n_tokens"]
+            response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
+                                                 ["comment"],
+                                                 GEN_QUESTION_TEMPERATURE)
+            perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following "
+                                      "Speaking Part 1 question: '" + question + "'")
+            token_count = count_tokens(perfect_answer_message)["n_tokens"]
+            response['perfect_answer'] = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT,
+                                                                   perfect_answer_message,
+                                                                   token_count,
+                                                                   None,
+                                                                   GEN_QUESTION_TEMPERATURE)
+            response['transcript'] = answer
+            response['fixed_text'] = get_speaking_corrections(answer)
+            return response
+        else:
+            return {
+                "comment": "The audio recorded does not contain enough english words to be graded.",
+                "overall": 0,
+                "task_response": {
+                    "Fluency and Coherence": 0,
+                    "Lexical Resource": 0,
+                    "Grammatical Range and Accuracy": 0,
+                    "Pronunciation": 0
+                }
+            }
+    except Exception as e:
+        os.remove(sound_file_name)
+        return str(e), 400
+
+
+@app.route('/speaking_task_1', methods=['GET'])
+@jwt_required()
+def get_speaking_task_1_question():
+    difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+    try:
+        gen_sp1_question = "Craft a thought-provoking question of " + difficulty + " difficulty for IELTS Speaking Part 1 that encourages candidates to delve deeply " \
+                           "into personal experiences, preferences, or insights on diverse topics. Instruct the candidate to offer " \
+                           "not only detailed descriptions but also provide nuanced explanations, examples, or anecdotes to enrich " \
+                           "their response. Make sure that the generated question does not contain forbidden subjects in muslim countries." \
+                           "Provide your response in this json format: {'topic': 'topic','question': 'question'}"
+        token_count = count_tokens(gen_sp1_question)["n_tokens"]
+        response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp1_question, token_count, GEN_FIELDS,
+                                             GEN_QUESTION_TEMPERATURE)
+        response["type"] = 1
+        response["difficulty"] = difficulty
+        return response
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/speaking_task_2', methods=['POST'])
+@jwt_required()
+def grade_speaking_task_2():
+    delete_files_older_than_one_day(AUDIO_FILES_PATH)
+    sound_file_name = AUDIO_FILES_PATH + str(uuid.uuid4())
+    try:
+        data = request.get_json()
+        question = data.get('question')
+        answer_firebase_path = data.get('answer')
+
+        download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name)
+        answer = speech_to_text(sound_file_name)
+        if has_10_words(answer):
+            message = ("Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a "
+                       "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
+                       "assign a score of 0 if the response fails to address the question. Additionally, provide "
+                       "detailed commentary highlighting both strengths and weaknesses in the response. Present your "
+                       "evaluation in JSON format with "
+                       "the following structure: {'comment': 'comment about answer quality', 'overall': 0.0, "
+                       "'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, 'Grammatical Range "
+                       "and Accuracy': 0.0, "
+                       "'Pronunciation': 0.0}}\n Question: '" + question + "' \n Answer: '" + answer + "'")
+            token_count = count_tokens(message)["n_tokens"]
+            response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
+                                                 ["comment"],
+                                                 GEN_QUESTION_TEMPERATURE)
+            perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following "
+                                      "Speaking Part 2 question: '" + question + "'")
+            token_count = count_tokens(perfect_answer_message)["n_tokens"]
+            response['perfect_answer'] = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT,
+                                                                   perfect_answer_message,
+                                                                   token_count,
+                                                                   None,
+                                                                   GEN_QUESTION_TEMPERATURE)
+            response['transcript'] = answer
+            response['fixed_text'] = get_speaking_corrections(answer)
+            return response
+        else:
+            return {
+                "comment": "The audio recorded does not contain enough english words to be graded.",
+                "overall": 0,
+                "task_response": {
+                    "Fluency and Coherence": 0,
+                    "Lexical Resource": 0,
+                    "Grammatical Range and Accuracy": 0,
+                    "Pronunciation": 0
+                }
+            }
+    except Exception as e:
+        os.remove(sound_file_name)
+        return str(e), 400
+
+
+@app.route('/speaking_task_2', methods=['GET'])
+@jwt_required()
+def get_speaking_task_2_question():
+    difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+    try:
+        gen_sp2_question = "Create a question of " + difficulty + " difficulty for IELTS Speaking Part 2 that encourages candidates to narrate a personal experience " \
+                           "or story related to a randomly selected topic. Include 3 prompts that guide the candidate to describe " \
+                           "specific aspects of the experience, such as details about the situation, their actions, and the " \
+                           "reasons it left a lasting impression. Make sure that the generated question does not contain forbidden subjects in muslim countries." \
+                           "Provide your response in this json format: {'topic': 'topic','question': 'question', " \
+                           "'prompts': ['prompt_1', 'prompt_2', 'prompt_3']}"
+        token_count = count_tokens(gen_sp2_question)["n_tokens"]
+        response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp2_question, token_count, GEN_FIELDS,
+                                             GEN_QUESTION_TEMPERATURE)
+        response["type"] = 2
+        response["difficulty"] = difficulty
+        return response
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/speaking_task_3', methods=['GET'])
+@jwt_required()
+def get_speaking_task_3_question():
+    difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+    try:
+        gen_sp3_question = "Formulate a set of 3 questions of " + difficulty + " difficulty for IELTS Speaking Part 3 that encourage candidates to engage in a " \
+                           "meaningful discussion on a particular topic. Provide inquiries, ensuring " \
+                           "they explore various aspects, perspectives, and implications related to the topic. " \
+                           "Make sure that the generated question does not contain forbidden subjects in muslim countries." \
+                           "Provide your response in this json format: {'topic': 'topic','questions': ['question', " \
+                           "'question', 'question']}"
+        token_count = count_tokens(gen_sp3_question)["n_tokens"]
+        response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_sp3_question, token_count, GEN_FIELDS,
+                                             GEN_QUESTION_TEMPERATURE)
+        # Remove the numbers from the questions only if the string starts with a number
+        response["questions"] = [re.sub(r"^\d+\.\s*", "", question) if re.match(r"^\d+\.", question) else question for
+                                 question in response["questions"]]
+        response["type"] = 3
+        response["difficulty"] = difficulty
+        return response
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/speaking_task_3', methods=['POST'])
+@jwt_required()
+def grade_speaking_task_3():
+    delete_files_older_than_one_day(AUDIO_FILES_PATH)
+    try:
+        data = request.get_json()
+        answers = data.get('answers')
+        text_answers = []
+        perfect_answers = []
+        for item in answers:
+            sound_file_name = AUDIO_FILES_PATH + str(uuid.uuid4())
+            download_firebase_file(FIREBASE_BUCKET, item["answer"], sound_file_name)
+            answer_text = speech_to_text(sound_file_name)
+            text_answers.append(answer_text)
+            item["answer"] = answer_text
+            os.remove(sound_file_name)
+            if not has_10_words(answer_text):
+                return {
+                    "comment": "The audio recorded does not contain enough english words to be graded.",
+                    "overall": 0,
+                    "task_response": {
+                        "Fluency and Coherence": 0,
+                        "Lexical Resource": 0,
+                        "Grammatical Range and Accuracy": 0,
+                        "Pronunciation": 0
+                    }
+                }
+            perfect_answer_message = ("Provide a perfect answer according to ielts grading system to the following "
+                                      "Speaking Part 3 question: '" + item["question"] + "'")
+            token_count = count_tokens(perfect_answer_message)["n_tokens"]
+            perfect_answers.append(make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT,
+                                                             perfect_answer_message,
+                                                             token_count,
+                                                             None,
+                                                             GEN_QUESTION_TEMPERATURE))
+        message = (
+            "Evaluate the given Speaking Part 2 response based on the IELTS grading system, ensuring a "
+            "strict assessment that penalizes errors. Deduct points for deviations from the task, and "
+            "assign a score of 0 if the response fails to address the question. Additionally, provide detailed "
+            "commentary highlighting both strengths and weaknesses in the response."
+            "\n\n The questions and answers are: \n\n'")
+
+        formatted_text = ""
+        for i, entry in enumerate(answers, start=1):
+            formatted_text += f"**Question {i}:**\n{entry['question']}\n\n"
+            formatted_text += f"**Answer {i}:**\n{entry['answer']}\n\n"
+
+        message += formatted_text
+        message += (
+            "'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', "
+            "'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, "
+            "'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}")
+
+        token_count = count_tokens(message)["n_tokens"]
+        response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
+                                             ["comment"],
+                                             GEN_QUESTION_TEMPERATURE)
+        for i, answer in enumerate(perfect_answers, start=1):
+            response['perfect_answer_' + str(i)] = answer
+
+        for i, answer in enumerate(text_answers, start=1):
+            response['transcript_' + str(i)] = answer
+            response['fixed_text_' + str(i)] = get_speaking_corrections(answer)
+        return response
+    except Exception as e:
+        return str(e), 400
+
+
+@app.route('/speaking', methods=['POST'])
+@jwt_required()
+def save_speaking():
+    try:
+        data = request.get_json()
+        exercises = data.get('exercises')
+        minTimer = data.get('minTimer', SPEAKING_MIN_TIMER_DEFAULT)
+        template = getSpeakingTemplate()
+        template["minTimer"] = minTimer
+
+        if minTimer < SPEAKING_MIN_TIMER_DEFAULT:
+            template["variant"] = ExamVariant.PARTIAL.value
+        else:
+            template["variant"] = ExamVariant.FULL.value
+
+        id = str(uuid.uuid4())
+        app.logger.info('Received request to save speaking with id: ' + id)
+        thread_event.set()
+        thread = threading.Thread(
+            target=create_videos_and_save_to_db,
+            args=(exercises, template, id),
+            name=("thread-save-speaking-" + id)
+        )
+        thread.start()
+        app.logger.info('Started thread to save speaking. Thread: ' + thread.getName())
+
+        # Return response without waiting for create_videos_and_save_to_db to finish
+        return {**template, "id": id}
+    except Exception as e:
+        return str(e)
+
+
+@app.route("/speaking/generate_speaking_video", methods=['POST'])
+@jwt_required()
+def generate_speaking_video():
+    try:
+        data = request.get_json()
+        avatar = data.get("avatar", random.choice(list(AvatarEnum)).value)
+        sp1_result = create_video(data["question"], avatar)
+        if sp1_result is not None:
+            sound_file_path = VIDEO_FILES_PATH + sp1_result
+            firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp1_result
+            url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+            sp1_video_path = firebase_file_path
+            sp1_video_url = url
+
+            return {
+                "text": data["question"],
+                "prompts": data["prompts"] if "prompts" in data else [],
+                "title": data["topic"],
+                "video_url": sp1_video_url,
+                "video_path": sp1_video_path,
+                "type": "speaking",
+                "id": uuid.uuid4()
+            }
+        else:
+            app.logger.error("Failed to create video for part 1 question: " + data["question"])
+            return str("Failed to create video for part 1 question: " + data["question"])
+
+    except Exception as e:
+        return str(e)
+
+
+@app.route("/speaking/generate_interactive_video", methods=['POST'])
+@jwt_required()
+def generate_interactive_video():
+    try:
+        data = request.get_json()
+        sp3_questions = []
+        avatar = data.get("avatar", random.choice(list(AvatarEnum)).value)
+
+        app.logger.info('Creating videos for speaking part 3')
+        for question in data["questions"]:
+            result = create_video(question, avatar)
+            if result is not None:
+                sound_file_path = VIDEO_FILES_PATH + result
+                firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + result
+                url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+                video = {
+                    "text": question,
+                    "video_path": firebase_file_path,
+                    "video_url": url
+                }
+                sp3_questions.append(video)
+            else:
+                app.app.logger.error("Failed to create video for part 3 question: " + question)
+
+        return {
+            "prompts": sp3_questions,
+            "title": data["topic"],
+            "type": "interactiveSpeaking",
+            "id": uuid.uuid4()
+        }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/reading_passage_1', methods=['GET'])
+@jwt_required()
+def get_reading_passage_1_question():
+    try:
+        # Extract parameters from the URL query string
+        topic = request.args.get('topic', default=random.choice(topics))
+        req_exercises = request.args.getlist('exercises')
+        difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+
+        if (len(req_exercises) == 0):
+            req_exercises = random.sample(READING_EXERCISE_TYPES, 2)
+
+        number_of_exercises_q = divide_number_into_parts(TOTAL_READING_PASSAGE_1_EXERCISES, len(req_exercises))
+
+        passage = generate_reading_passage(QuestionType.READING_PASSAGE_1, topic)
+        app.logger.info("Generated passage: " + str(passage))
+        start_id = 1
+        exercises = generate_reading_exercises(passage["text"], req_exercises, number_of_exercises_q, start_id, difficulty)
+        return {
+            "exercises": exercises,
+            "text": {
+                "content": passage["text"],
+                "title": passage["title"]
+            },
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/reading_passage_2', methods=['GET'])
+@jwt_required()
+def get_reading_passage_2_question():
+    try:
+        # Extract parameters from the URL query string
+        topic = request.args.get('topic', default=random.choice(topics))
+        req_exercises = request.args.getlist('exercises')
+        difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+
+        if (len(req_exercises) == 0):
+            req_exercises = random.sample(READING_EXERCISE_TYPES, 2)
+
+        number_of_exercises_q = divide_number_into_parts(TOTAL_READING_PASSAGE_2_EXERCISES, len(req_exercises))
+
+        passage = generate_reading_passage(QuestionType.READING_PASSAGE_2, topic)
+        app.logger.info("Generated passage: " + str(passage))
+        start_id = 14
+        exercises = generate_reading_exercises(passage["text"], req_exercises, number_of_exercises_q, start_id, difficulty)
+        return {
+            "exercises": exercises,
+            "text": {
+                "content": passage["text"],
+                "title": passage["title"]
+            },
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/reading_passage_3', methods=['GET'])
+@jwt_required()
+def get_reading_passage_3_question():
+    try:
+        # Extract parameters from the URL query string
+        topic = request.args.get('topic', default=random.choice(topics))
+        req_exercises = request.args.getlist('exercises')
+        difficulty = request.args.get("difficulty", default=random.choice(difficulties))
+
+        if (len(req_exercises) == 0):
+            req_exercises = random.sample(READING_EXERCISE_TYPES, 2)
+
+        number_of_exercises_q = divide_number_into_parts(TOTAL_READING_PASSAGE_3_EXERCISES, len(req_exercises))
+
+        passage = generate_reading_passage(QuestionType.READING_PASSAGE_3, topic)
+        app.logger.info("Generated passage: " + str(passage))
+        start_id = 27
+        exercises = generate_reading_exercises(passage["text"], req_exercises, number_of_exercises_q, start_id, difficulty)
+        return {
+            "exercises": exercises,
+            "text": {
+                "content": passage["text"],
+                "title": passage["title"]
+            },
+            "difficulty": difficulty
+        }
+    except Exception as e:
+        return str(e)
+
+
+# THE SAVING OF READING IS DONE WITHOUT THE API ON THE FRONTEND
+# @app.route('/reading', methods=['POST'])
+# @jwt_required()
+# def save_reading_passage():
+#     try:
+#         data = request.get_json()
+#         parts = data.get('parts')
+#         template = getReadingTemplate()
+#         template["parts"] = parts
+#         id = str(uuid.uuid4())
+#         (result, id) = save_to_db_with_id("reading", template, id)
+#         if result:
+#             return {**template, "id": id}
+#         else:
+#             raise Exception("Failed to save reading: " + template)
+#     except Exception as e:
+#         return str(e)
+
+
+@app.route('/level', methods=['GET'])
+@jwt_required()
+def get_level_exam():
+    try:
+        number_of_exercises = 25
+        exercises = gen_multiple_choice_level(number_of_exercises)
+        return {
+            "exercises": [exercises],
+            "isDiagnostic": False,
+            "minTimer": 25,
+            "module": "level"
+        }
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/fetch_tips', methods=['POST'])
+@jwt_required()
+def fetch_answer_tips():
+    try:
+        data = request.get_json()
+        context = data.get('context')
+        question = data.get('question')
+        answer = data.get('answer')
+        correct_answer = data.get('correct_answer')
+        messages = get_question_tips(question, answer, correct_answer, context)
+        token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
+                             map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
+        response = make_openai_call(GPT_3_5_TURBO, messages, token_count, None, TIPS_TEMPERATURE)
+
+        if isinstance(response, str):
+            response = re.sub(r"^[a-zA-Z0-9_]+\:\s*", "", response)
+
+        return response
+    except Exception as e:
+        return str(e)
+
+
+@app.route('/grading_summary', methods=['POST'])
+@jwt_required()
+def grading_summary():
+    # Body Format
+    # {'sections': Array of {'code': key, 'name': name, 'grade': grade}}
+    # Output Format
+    # {'sections': Array of {'code': key, 'name': name, 'grade': grade, 'evaluation': evaluation, 'suggestions': suggestions}}
+    try:
+        return calculate_grading_summary(request.get_json())
+    except Exception as e:
+        return str(e)
 
 
 if __name__ == '__main__':

bs_playground.py

@@ -1,80 +0,0 @@
-import streamlit as st
-import openai
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-openai.api_key = os.getenv("OPENAI_API_KEY")
-
-
-def generate_summarizer(
-        temperature,
-        question_type,
-        content
-):
-    res = openai.ChatCompletion.create(
-        model="gpt-3.5-turbo",
-        temperature=float(temperature),
-        messages=[
-            {
-                "role": "system",
-                "content": "You are a IELTS exam question generation program.",
-            },
-            {
-                "role": "system",
-                "content": f"Generate a simple {question_type} for the following text: {content}",
-            },
-            {
-                "role": "system",
-                "content": "Please provide a JSON object response with the overall grade and breakdown grades, "
-                           "formatted as follows: {'overall': 7.0, 'task_response': {'Task Achievement': 8.0, "
-                           "'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, 'Grammatical Range and Accuracy': "
-                           "6.0}}",
-            },
-        ],
-    )
-    return res["choices"][0]["message"]["content"]
-
-
-# Set the application title
-st.title("GPT-3.5 IELTS Question Generation Program")
-
-# qt_col, q_col = st.columns(2)
-
-# Selection box to select the question type
-# with qt_col:
-question_type = st.selectbox(
-    "What is the question type?",
-    (
-        "Writing Task 2",
-    ),
-)
-
-# Provide the input area for question to be answered
-# with q_col:
-content = st.text_area("Enter the content:", height=100)
-
-# Initiate two columns for section to be side-by-side
-# col1, col2 = st.columns(2)
-
-# Slider to control the model hyperparameter
-# with col1:
-# token = st.slider("Token", min_value=0.0,
-#                  max_value=2000.0, value=1000.0, step=1.0)
-temp = st.slider("Temperature", min_value=0.0,
-                 max_value=1.0, value=0.7, step=0.01)
-# top_p = st.slider("Top_p", min_value=0.0, max_value=1.0, value=0.9, step=0.01)
-# f_pen = st.slider("Frequency Penalty", min_value=-1.0,
-#                  max_value=1.0, value=0.5, step=0.01)
-
-# Showing the current parameter used for the model
-# with col2:
-with st.expander("Current Parameter"):
-    #    st.write("Current Token :", token)
-    st.write("Current Temperature :", temp)
-#    st.write("Current Nucleus Sampling :", top_p)
-#    st.write("Current Frequency Penalty :", f_pen)
-
-# Creating button for execute the text summarization
-if st.button("Grade"):
-    st.write(generate_summarizer(temp, question_type, content))

download-audio/placeholder.txt

@@ -0,0 +1 @@
+THIS FILE ONLY EXISTS TO KEEP THIS FOLDER IN THE REPO

download-video/placeholder.txt

@@ -0,0 +1 @@
+THIS FILE ONLY EXISTS TO KEEP THIS FOLDER IN THE REPO

firebase-configs/mti-ielts-626a2dcf6091.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "mti-ielts",
+  "private_key_id": "626a2dcf60916a1b5011f388495b8f9c4fc065ef",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDuaLgLNa5yb5LI\nPZYa7qav0URgCF7miK3dUXIBoABQ+U6y1LwdsIiJqHZ4Cm2lotTqeTGOIV83PuA6\n9H/TwnvsHH8jilmsPxO5OX7AyZSDPvN45nJrgQ21RKZCYQGVetBMGhclCRbYFraS\nE6X/p6gSOpSqZ5fLz8BbdCMfib6HSfDmBkYTK42X6d2eNNwLM1wLbE8RmCGwRATC\nQFfMhjlvQcSJ1EDMfkMUUE9U/ux77wfHqs1d+7utVcQTIMFAP9fo1ynJlwp8D1HQ\ntalB6kkpuDQetUR0A1FHMMJekhmuRDUMfokX1F9JfUjR0OetuD3KEH5y2asxC2+0\n8JYcwbvlAgMBAAECggEAKaaW3LJ8rxZp/NyxkDP4YAf9248q0Ti4s00qzzjeRUdA\n5gI/eSphuDb7t34O6NyZOPuCWlPfOB4ee35CpMK59qaF2bYuc2azseznBZRSA1no\nnEsaW0i5Fd2P9FHRPoWtxVXbjEdZu9e//qY7Hn5yYPjmBx1BCkTZ1MBl8HkWlbjR\nbu18uveg5Vg6Wc+rnPmH/gMRLLpq9iQBpzXWT8Mj+k48O8GnW6v8S3R027ymqUou\n3W5b69xDGn0nwxgLIVzdxjoo7RnpjD3mP0x4faiBhScVgFhwZP8hqBeVyqbV5dMh\nfF+p9zLOeilFLJEjH1lZbZAb8wwP23LozIXJWFG3oQKBgQD6COCJ7hNSx9/AzDhO\nh73hKH/KSOJtxHc8795hcZjy9HJkoM45Fm7o2QGZzsZmV+N6VU0BjoDQAyftCq+G\ndIX0wcAGJIsLuQ9K00WI2hn7Uq1gjUl0d9XEorogKa1ZNTLL/9By/xnA7sEpI6Ng\nIsKQ4R2CfqNFU4bs1nyKWCWudQKBgQD0GNYwZt3xV2YBATVYsrvg1OGO/tmkCJ8Y\nLOdM0L+8WMCgw0uQcNFF9uqq6/oFgq7tOvpeZDsY8onRy55saaMT+Lr4xs0sj5B0\ns5Hqc0L37tdXXXXEne8WABMBF9injNgNbAm9W0kqME2Stc53OJQPj2DBdYxWSr8v\n36imCwoJsQKBgH0BBSlQQo7naKFeOGRijvbLpZ//clzIlYh8r+Rtw7brqWlPz+pQ\noeB95cP80coG9K6LiPVXRmU4vrRO3FRPW01ztEod6PpSaifRmnkB+W1h91ZHLMsy\nwkgNxxofXBA2fY/p9FAZ48lGVIH51EtS9Y0zTuqX347gZJtx3E/aI/SlAoGBAJer\nCwM+F2+K352GM7BuNiDoBVLFdVPf64Ko+/sVxdzwxJffYQdZoh634m3bfBmKbsiG\nmeSmoLXKlenefAxewu544SwM0pV6isaIgQTNI3JMXE8ziiZl/5WK7EQEniDVebU1\nSQP4QYjORJUBFE2twQm+C9+I+27uuMa1UOQC/fSxAoGBANuWloacqGfws6nbHvqF\nLZKlkKNPI/0sC+6VlqjoHn5LQz3lcFM1+iKSQIGJvJyru2ODgv2Lmq2W+cx+HMeq\n0BSetK4XtalmO9YflH7uMgvOEVewf4uJ2d+4I1pbY9aI1gHaZ1EUiiy6Ds4kAK8s\nTQqp88pfTbOnkdJBVi0AWs5B\n-----END PRIVATE KEY-----\n",
+  "client_email": "firebase-adminsdk-dyg6p@mti-ielts.iam.gserviceaccount.com",
+  "client_id": "104980563453519094431",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-dyg6p%40mti-ielts.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

firebase-configs/storied-phalanx-349916.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "storied-phalanx-349916",
+  "private_key_id": "c9e05f6fe413b1031a71f981160075ff4b044444",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDdgavFB63nMHyb\n38ncwijTrUmqU9UyzNJ8wlZCWAWuoz25Gng988fkKNDXnHY+ap9esHyNYg9IdSA7\nAuZeHpzTZmKiWZzFWq61KWSTgIn1JwKHGHJJdmVhTYfCe9I51cFLa5q2lTFzJ0ce\nbP7/X/7kw53odgva+M8AhDTbe60akpemgZc+LFwO0Abm7erH2HiNyjoNZzNw525L\n933PCaQwhZan04s1u0oRdVlBIBwMk+J0ojgVEpUiJOzF7gkN+UpDXujalLYdlR4q\nhkGgScXQhDYJkECC3GuvOnEo1YXGNjW9D73S6sSH+Lvqta4wW1+sTn0kB6goiQBI\n7cA1G6x3AgMBAAECggEAZPMwAX/adb7XS4LWUNH8IVyccg/63kgSteErxtiu3kRv\nYOj7W+C6fPVNGLap/RBCybjNSvIh3PfkVICh1MtG1eGXmj4VAKyvaskOmVq/hQbe\nVAuEKo7W7V2UPcKIsOsGSQUlYYjlHIIOG4O5Q1HQrRmp4cPK62Txkl6uaEkZPz4u\nbvIK2BJI8aHRwxE3Phw09blwlLqQQQ8nrhK29x5puaN+ft++IlzIOVsLz+n4kTdB\n6qkG/dhenn3K8o3+NkmSN6eNRbdJd36zXTo4Oatbvqb7r0E8vYn/3Llawo2X75zn\nec7jMHrOmcwtiu9H3PsrTWtzdSjxPHy0UtEn1HWK4QKBgQD+c/V8tAvbaUGVoZf6\ntKtDSKF6IHuY2vUO33v950mVdjrTursqOG2d+SLfSnKpc+sjDlj7/S5u4uRP+qUN\ng1rb2U7oIA7tsDa2ZTSkIx6HkPUzS+fBOxELLrbgMoJ2RLzgkiPhS95YgXJ/rYG5\nWQTehzCT5roes0RvtgM0gl3EhQKBgQDe2m7PRIU4g3RJ8HTx92B4ja8W9FVCYDG5\nPOAdZB8WB6Bvu4BJHBDLr8vDi930pKj+vYObRqBDQuILW4t8wZQJ834dnoq6EpUz\nhbVEURVBP4A/nEHrQHfq0Lp+cxThy2rw7obRQOLPETtC7p3WFgSHT6PRTcpGzCCX\n+76a30yrywKBgC/5JNtyBppDaf4QDVtTHMb+tpMT9LmI7pLzR6lDJfhr5gNtPURk\nhyY1hoGaw6t3E2n0lopL3alCVdFObDfz//lbKylQggAGLQqOYjJf/K2KgvA862Df\nBgOZtxjl7PrnUsT0SJd9elotbazsxXxwcB6UVnBMG+MV4V0+b7RCr/MRAoGBAIfp\nTcVIs7roqOZjKN9dEE/VkR/9uXW2tvyS/NfP9Ql5c0ZRYwazgCbJOwsyZRZLyek6\naWYsp5b91mA435QhdwiuoI6t30tmA+qdNBTLIpxdfvjMcoNoGPpzfBmcU/L1HW58\n+mnqGalRiAPlBQvI99ASKQWAXMnaulIWrYNEhj0LAoGBALi+QZ2pp+hDeC59ezWr\nbP1zbbONceHKGgJcevChP2k1OJyIOIqmBYeTuM4cPc5ofZYQNaMC31cs8SVeSRX1\nNTxQZmvCjMyTe/WYWYNFXdgkVz4egFXbeochCGzMYo57HV1PCkPBrARRZO8OfdDD\n8sDu//ohb7nCzceEI0DnWs13\n-----END PRIVATE KEY-----\n",
+  "client_email": "firebase-adminsdk-3ml0u@storied-phalanx-349916.iam.gserviceaccount.com",
+  "client_id": "114163760341944984396",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-3ml0u%40storied-phalanx-349916.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

generate_speaking_questions.py

@@ -0,0 +1,180 @@
+import random
+
+from helper.firebase_helper import upload_file_firebase_get_url
+from helper.heygen_api import create_video
+import os
+import uuid
+
+import firebase_admin
+from firebase_admin import credentials, firestore
+
+from dotenv import load_dotenv
+
+from heygen.AvatarEnum import AvatarEnum
+
+load_dotenv()
+
+# Initialize Firebase Admin SDK
+cred = credentials.Certificate(os.getenv("GOOGLE_APPLICATION_CREDENTIALS"))
+
+firebase_admin.initialize_app(cred)
+
+FIREBASE_BUCKET = 'storied-phalanx-349916.appspot.com'
+VIDEO_FILES_PATH = 'download-video/'
+FIREBASE_SPEAKING_VIDEO_FILES_PATH = 'speaking_videos/'
+
+# PART 1
+sp1_video_url = ""
+sp1_video_path = ""
+
+# sp1_questions_json = {
+#   "topic": "Hobbies and Interests",
+#   "question": "What do you like to do in your free time? Do you have any hobbies or interests? Are there any new hobbies you would like to try in the future?"
+# }
+sp1_questions_json = {
+    "topic": "Hobbies and Interests",
+    "question": "Do you have any hobbies or interests that you enjoy in your free time? Please describe one of your favorite hobbies and explain why you like it."
+}
+
+sp1_result = create_video(sp1_questions_json["question"], random.choice(list(AvatarEnum)))
+if sp1_result is not None:
+    sound_file_path = VIDEO_FILES_PATH + sp1_result
+    firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp1_result
+    url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+    sp1_video_path = firebase_file_path
+    sp1_video_url = url
+else:
+    print("Failed to create video for question: " + sp1_questions_json["question"])
+
+# PART 2
+sp2_video_url = ""
+sp2_video_path = ""
+
+# sp2_questions_json = {
+#   "topic": "New Environment",
+#   "question": "Describe an occasion when you had to adapt to a new environment."
+# }
+
+sp2_questions_json = {
+    "topic": "Travel and Adventure",
+    "question": "Describe an occasion when you had to adapt to a new environment.",
+    "prompts": [
+        "Please include details about the destination",
+        "What you did there",
+        "why this trip was memorable for you."
+    ]
+}
+
+sp2_result = create_video(sp2_questions_json["question"], random.choice(list(AvatarEnum)))
+if sp2_result is not None:
+    sound_file_path = VIDEO_FILES_PATH + sp2_result
+    firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp2_result
+    url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+    sp2_video_path = firebase_file_path
+    sp2_video_url = url
+else:
+    print("Failed to create video for question: " + sp2_questions_json["question"])
+
+# PART 3
+# 1
+# sp3_questions_json = {
+#     "topic": "Technology and Society",
+#     "questions": [
+#         "How do you think technology has affected the way people communicate with each other in today's society?",
+#         "In what ways has the use of smartphones and social media platforms changed the dynamics of personal relationships?",
+#         "Some argue that technology has made communication more convenient, while others worry that it has led to a decline in face-to-face interactions. What's your perspective on this matter, and how do you think it might impact future generations?"
+#     ]
+# }
+
+# 2
+# sp3_questions_json = {
+#     "topic": "Environmental Concerns",
+#     "questions": [
+#         "What do you believe are the most pressing environmental issues facing the world today?",
+#         "In your opinion, what role should governments play in addressing environmental concerns?",
+#         "Many people argue that individual actions, like recycling and conserving energy, are not sufficient to address environmental issues. What's your perspective on this? How can individuals contribute to solving these problems?"
+#     ]
+# }
+
+# 3
+# sp3_questions_json = {
+#   "topic": "Education and Technology",
+#   "questions": [
+#     "How has technology impacted the way students learn in today's classrooms?",
+#     "Do you think traditional textbooks will become obsolete in the future due to digital learning materials? Why or why not?",
+#     "What are the potential advantages and disadvantages of online courses and e-learning platforms?"
+#   ]
+# }
+
+# 4
+sp3_questions_json = {
+  "topic": "Cultural Diversity",
+  "questions": [
+    "How can exposure to different cultures benefit individuals and society as a whole?",
+    "What challenges might arise in culturally diverse communities, and how can they be addressed?",
+    "Do you think it's important for schools to incorporate cultural diversity into their curriculum? Why or why not?"
+  ]
+}
+
+questions = []
+avatar = random.choice(list(AvatarEnum))
+for question in sp3_questions_json["questions"]:
+    result = create_video(question, avatar)
+    if result is not None:
+        sound_file_path = VIDEO_FILES_PATH + result
+        firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + result
+        url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+        video = {
+            "text": question,
+            "video_path": firebase_file_path,
+            "video_url": url
+        }
+        questions.append(video)
+    else:
+        print("Failed to create video for question: " + question)
+
+# BUILD OBJECT TO SAVE
+if len(questions) == len(sp3_questions_json["questions"]):
+    speaking_module_to_insert = {
+        "exercises": [
+            {
+                "id": str(uuid.uuid4()),
+                "prompts": [],
+                "text": sp1_questions_json["question"],
+                "title": sp1_questions_json["topic"],
+                "video_url": sp1_video_url,
+                "video_path": sp1_video_path,
+                "type": "speaking"
+            },
+            {
+                "id": str(uuid.uuid4()),
+                "prompts": sp2_questions_json["prompts"],
+                "text": sp2_questions_json["question"],
+                "title": sp2_questions_json["topic"],
+                "video_url": sp2_video_url,
+                "video_path": sp2_video_path,
+                "type": "speaking"
+            },
+            {
+                "id": str(uuid.uuid4()),
+                "prompts": questions,
+                "text": "Listen carefully and respond.",
+                "title": sp3_questions_json["topic"],
+                "type": "interactiveSpeaking"
+            }
+        ],
+        "isDiagnostic": True,
+        "minTimer": 14,
+        "module": "speaking"
+    }
+
+    db = firestore.client()
+
+    # JSON data to insert
+
+    # Add the JSON data to Firestore
+    collection_ref = db.collection('speaking')
+    document_ref = collection_ref.add(speaking_module_to_insert)
+    print(f"Document added with ID: {document_ref}")
+else:
+    print("Array sizes do not match. Video uploading failing is probably the cause.")

helper/ExamVariant.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class ExamVariant(Enum):
+    FULL = "full"
+    PARTIAL = "partial"

helper/api_messages.py

@@ -0,0 +1,444 @@
+from enum import Enum
+
+from typing import List
+
+
+class QuestionType(Enum):
+    LISTENING_SECTION_1 = "Listening Section 1"
+    LISTENING_SECTION_2 = "Listening Section 2"
+    LISTENING_SECTION_3 = "Listening Section 3"
+    LISTENING_SECTION_4 = "Listening Section 4"
+    WRITING_TASK_1 = "Writing Task 1"
+    WRITING_TASK_2 = "Writing Task 2"
+    SPEAKING_1 = "Speaking Task Part 1"
+    SPEAKING_2 = "Speaking Task Part 2"
+    READING_PASSAGE_1 = "Reading Passage 1"
+    READING_PASSAGE_2 = "Reading Passage 2"
+    READING_PASSAGE_3 = "Reading Passage 3"
+
+class ExerciseType(Enum):
+    MULTIPLE_CHOICE = "multiple choice"
+
+
+def get_grading_messages(question_type: QuestionType, question: str, answer: str, context: str = None):
+    if QuestionType.WRITING_TASK_1 == question_type:
+        messages = [
+            {
+                "role": "user",
+                "content": "You are a IELTS examiner.",
+            },
+            {
+                "role": "user",
+                "content": f"The question you have to grade is of type Writing Task 1 and is the following: {question}",
+            }
+        ]
+
+        if not (context is None or context == ""):
+            messages.append({
+                "role": "user",
+                "content": f"To grade the previous question, bear in mind the following context: {context}",
+            })
+
+        messages.extend([
+            {
+                "role": "user",
+                "content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
+                           "with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, "
+                           "'task_response': {'Task Achievement': 8.0, 'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, "
+                           "'Grammatical Range and Accuracy': 6.0}}",
+            },
+            {
+                "role": "user",
+                "content": "Example output: { 'comment': 'Overall, the response is good but there are some areas that need "
+                           "improvement.\n\nIn terms of Task Achievement, the writer has addressed all parts of the question "
+                           "and has provided a clear opinion on the topic. However, some of the points made are not fully "
+                           "developed or supported with examples.\n\nIn terms of Coherence and Cohesion, there is a clear "
+                           "structure to the response with an introduction, body paragraphs and conclusion. However, there "
+                           "are some issues with cohesion as some sentences do not flow smoothly from one to another.\n\nIn "
+                           "terms of Lexical Resource, there is a good range of vocabulary used throughout the response and "
+                           "some less common words have been used effectively.\n\nIn terms of Grammatical Range and Accuracy, "
+                           "there are some errors in grammar and sentence structure which affect clarity in places.\n\nOverall, "
+                           "this response would score a band 6.5.', 'overall': 6.5, 'task_response': "
+                           "{ 'Coherence and Cohesion': 6.5, 'Grammatical Range and Accuracy': 6.0, 'Lexical Resource': 7.0, "
+                           "'Task Achievement': 7.0}}",
+            },
+            {
+                "role": "user",
+                "content": f"Evaluate this answer according to ielts grading system: {answer}",
+            },
+        ])
+        return messages
+    elif QuestionType.WRITING_TASK_2 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are a IELTS examiner.",
+            },
+            {
+                "role": "user",
+                "content": f"The question you have to grade is of type Writing Task 2 and is the following: {question}",
+            },
+            {
+                "role": "user",
+                "content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, "
+                           "with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, "
+                           "'task_response': {'Task Achievement': 8.0, 'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, "
+                           "'Grammatical Range and Accuracy': 6.0}}",
+            },
+            {
+                "role": "user",
+                "content": "Example output: { 'comment': 'Overall, the response is good but there are some areas that need "
+                           "improvement.\n\nIn terms of Task Achievement, the writer has addressed all parts of the question "
+                           "and has provided a clear opinion on the topic. However, some of the points made are not fully "
+                           "developed or supported with examples.\n\nIn terms of Coherence and Cohesion, there is a clear "
+                           "structure to the response with an introduction, body paragraphs and conclusion. However, there "
+                           "are some issues with cohesion as some sentences do not flow smoothly from one to another.\n\nIn "
+                           "terms of Lexical Resource, there is a good range of vocabulary used throughout the response and "
+                           "some less common words have been used effectively.\n\nIn terms of Grammatical Range and Accuracy, "
+                           "there are some errors in grammar and sentence structure which affect clarity in places.\n\nOverall, "
+                           "this response would score a band 6.5.', 'overall': 6.5, 'task_response': "
+                           "{ 'Coherence and Cohesion': 6.5, 'Grammatical Range and Accuracy': 6.0, 'Lexical Resource': 7.0, "
+                           "'Task Achievement': 7.0}}",
+            },
+            {
+                "role": "user",
+                "content": f"Evaluate this answer according to ielts grading system: {answer}",
+            },
+        ]
+    elif QuestionType.SPEAKING_1 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are an IELTS examiner."
+            },
+            {
+                "role": "user",
+                "content": f"The question you need to grade is a Speaking Task Part 1 question, and it is as follows: {question}"
+            },
+            {
+                "role": "user",
+                "content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about answer "
+                           "quality will go here', 'overall': 7.0, 'task_response': {'Fluency and "
+                           "Coherence': 8.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}"
+            },
+            {
+                "role": "user",
+                "content": "Example output: {'comment': 'Comment about answer quality will go here', 'overall': 6.5, "
+                           "'task_response': {'Fluency and Coherence': 7.0, "
+                           "'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}"
+            },
+            {
+                "role": "user",
+                "content": "Please assign a grade of 0 if the answer provided does not address the question."
+            },
+            {
+                "role": "user",
+                "content": f"Assess this answer according to the IELTS grading system: {answer}"
+            },
+            {
+                "role": "user",
+                "content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, "
+                           "and Pronunciation when grading the response."
+            }
+        ]
+    elif QuestionType.SPEAKING_2 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are an IELTS examiner."
+            },
+            {
+                "role": "user",
+                "content": f"The question you need to grade is a Speaking Task Part 2 question, and it is as follows: {question}"
+            },
+            {
+                "role": "user",
+                "content": "Please provide your assessment using the following JSON format: {\"comment\": \"Comment about "
+                           "answer quality\", \"overall\": 7.0, \"task_response\": {\"Fluency and Coherence\": 8.0, \"Lexical "
+                           "Resource\": 6.5, \"Grammatical Range and Accuracy\": 7.5, \"Pronunciation\": 6.0}}"
+            },
+            {
+                "role": "user",
+                "content": "Example output: {\"comment\": \"The candidate has provided a clear response to the question "
+                           "and has given examples of how they spend their weekends. However, there are some issues with "
+                           "grammar and pronunciation that affect the overall score. In terms of fluency and coherence, "
+                           "the candidate speaks clearly and smoothly with only minor hesitations. They have also provided "
+                           "a well-organized response that is easy to follow. Regarding lexical resource, the candidate "
+                           "has used a range of vocabulary related to weekend activities but there are some errors in "
+                           "word choice that affect the meaning of their sentences. In terms of grammatical range and "
+                           "accuracy, the candidate has used a mix of simple and complex sentence structures but there "
+                           "are some errors in subject-verb agreement and preposition use. Finally, regarding pronunciation, "
+                           "the candidate's speech is generally clear but there are some issues with stress and intonation "
+                           "that make it difficult to understand at times.\", \"overall\": 6.5, \"task_response\": {\"Fluency "
+                           "and Coherence\": 7.0, \"Lexical Resource\": 6.5, \"Grammatical Range and Accuracy\": 7.0, "
+                           "\"Pronunciation\": 6.0}}"
+            },
+            {
+                "role": "user",
+                "content": "Please assign a grade of 0 if the answer provided does not address the question."
+            },
+            {
+                "role": "user",
+                "content": f"Assess this answer according to the IELTS grading system: {answer}"
+            },
+            {
+                "role": "user",
+                "content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, "
+                           "and Pronunciation when grading the response."
+            }
+        ]
+    else:
+        raise Exception("Question type not implemented: " + question_type.value)
+
+
+def get_speaking_grading_messages(answers: List):
+    messages = [
+        {
+            "role": "user",
+            "content": "You are an IELTS examiner."
+        },
+        {
+            "role": "user",
+            "content": "The exercise you need to grade is a Speaking Task, and it is has the following questions and answers:"
+        }
+    ]
+    for item in answers:
+        question = item["question"]
+        answer = item["answer_text"]
+        messages.append({
+            "role": "user",
+            "content": f"Question: {question}; Answer: {answer}"
+        })
+    messages.extend([
+        {
+            "role": "user",
+            "content": f"Assess this answer according to the IELTS grading system."
+        },
+        {
+            "role": "user",
+            "content": "Please provide your assessment using the following JSON format: {'comment': 'Comment about answer "
+                       "quality will go here', 'overall': 7.0, 'task_response': {'Fluency and "
+                       "Coherence': 8.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}"
+        },
+        {
+            "role": "user",
+            "content": "Example output: {'comment': 'Comment about answer quality will go here', 'overall': 6.5, "
+                       "'task_response': {'Fluency and Coherence': 7.0, "
+                       "'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0, 'Pronunciation': 6.0}}"
+        },
+        {
+            "role": "user",
+            "content": "Please assign a grade of 0 if the answer provided does not address the question."
+        },
+        {
+            "role": "user",
+            "content": "Remember to consider Fluency and Coherence, Lexical Resource, Grammatical Range and Accuracy, "
+                       "and Pronunciation when grading the response."
+        }
+    ])
+    return messages
+
+
+def get_question_gen_messages(question_type: QuestionType):
+    if QuestionType.LISTENING_SECTION_1 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are a IELTS program that generates questions for the exams.",
+            },
+            {
+                "role": "user",
+                "content": "Provide me with a transcript similar to the ones in ielts exam Listening Section 1. "
+                           "Create an engaging transcript simulating a conversation related to a unique type of service "
+                           "that requires getting the customer's details. Make sure to include specific details "
+                           "and descriptions to bring"
+                           "the scenario to life. After the transcript, please "
+                           "generate a 'form like' fill in the blanks exercise with 6 form fields (ex: name, date of birth)"
+                           " to fill related to the customer's details. Finally, "
+                           "provide the answers for the exercise. The response must be a json following this format: "
+                           "{ 'type': '<type of registration (ex: hotel, gym, english course, etc)>', "
+                           "'transcript': '<transcript of just the conversation about a registration of some sort, "
+                           "identify the person talking in each speech line>', "
+                           "'exercise': { 'form field': { '1': '<form field 1>', '2': '<form field 2>', "
+                           "'3': '<form field 3>', '4': '<form field 4>', "
+                           "'5': '<form field 5>', '6': '<form field 5>' }, "
+                           "'answers': {'1': '<answer to fill blank space in form field 1>', '2': '<answer to fill blank "
+                           "space in form field 2>', '3': '<answer to fill blank space in form field 3>', "
+                           "'4': '<answer to fill blank space in form field 4>', '5': '<answer to fill blank space in form field 5>',"
+                           " '6': '<answer to fill blank space in form field 6>'}}}",
+            }
+        ]
+    elif QuestionType.LISTENING_SECTION_2 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are a IELTS program that generates questions for the exams.",
+            },
+            {
+                "role": "user",
+                "content": "Provide me with a transcript similar to the ones in ielts exam Listening section 2. After the transcript, please "
+                           "generate a fill in the blanks exercise with 6 statements related to the text content. Finally, "
+                           "provide the answers for the exercise. The response must be a json following this format: "
+                           "{ 'transcript': 'transcript about some subject', 'exercise': { 'statements': { '1': 'statement 1 "
+                           "with a blank space to fill', '2': 'statement 2 with a blank space to fill', '3': 'statement 3 with a "
+                           "blank space to fill', '4': 'statement 4 with a blank space to fill', '5': 'statement 5 with a blank "
+                           "space to fill', '6': 'statement 6 with a blank space to fill' }, "
+                           "'answers': {'1': 'answer to fill blank space in statement 1', '2': 'answer to fill blank "
+                           "space in statement 2', '3': 'answer to fill blank space in statement 3', "
+                           "'4': 'answer to fill blank space in statement 4', '5': 'answer to fill blank space in statement 5',"
+                           " '6': 'answer to fill blank space in statement 6'}}}",
+            }
+        ]
+    elif QuestionType.LISTENING_SECTION_3 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are a IELTS program that generates questions for the exams.",
+            },
+            {
+                "role": "user",
+                "content": "Provide me with a transcript similar to the ones in ielts exam Listening section 3. After the transcript, please "
+                           "generate 4 multiple choice questions related to the text content. Finally, "
+                           "provide the answers for the exercise. The response must be a json following this format: "
+                           "{ 'transcript': 'generated transcript similar to the ones in ielts exam Listening section 3', "
+                           "'exercise': { 'questions': [ { 'question': "
+                           "'question 1', 'options': ['option 1', 'option 2', 'option 3', 'option 4'], 'answer': 1}, "
+                           "{'question': 'question 2', 'options': ['option 1', 'option 2', 'option 3', 'option 4'], "
+                           "'answer': 3}, {'question': 'question 3', 'options': ['option 1', 'option 2', 'option 3', "
+                           "'option 4'], 'answer': 0}, {'question': 'question 4', 'options': ['option 1', 'option 2', "
+                           "'option 3', 'option 4'], 'answer': 2}]}}",
+            }
+        ]
+    elif QuestionType.LISTENING_SECTION_4 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are a IELTS program that generates questions for the exams.",
+            },
+            {
+                "role": "user",
+                "content": "Provide me with a transcript similar to the ones in ielts exam Listening section 4. After the transcript, please "
+                           "generate 4 completion-type questions related to the text content to complete with 1 word. Finally, "
+                           "provide the answers for the exercise. The response must be a json following this format: "
+                           "{ 'transcript': 'generated transcript similar to the ones in ielts exam Listening section 4', "
+                           "'exercise': [ { 'question': 'question 1', 'answer': 'answer 1'}, "
+                           "{'question': 'question 2', 'answer': 'answer 2'}, {'question': 'question 3', 'answer': 'answer 3'}, "
+                           "{'question': 'question 4', 'answer': 'answer 4'}]}",
+            }
+        ]
+    elif QuestionType.WRITING_TASK_2 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are a IELTS program that generates questions for the exams.",
+            },
+            {
+                "role": "user",
+                "content": "The question you have to generate is of type Writing Task 2.",
+            },
+            {
+                "role": "user",
+                "content": "It is mandatory for you to provide your response with the question "
+                           "just with the following json format: {'question': 'question'}",
+            },
+            {
+                "role": "user",
+                "content": "Example output: { 'question': 'We are becoming increasingly dependent on computers. "
+                           "They are used in businesses, hospitals, crime detection and even to fly planes. What things will "
+                           "they be used for in the future? Is this dependence on computers a good thing or should we he more "
+                           "auspicious of their benefits?'}",
+            },
+            {
+                "role": "user",
+                "content": "Generate a question for IELTS exam Writing Task 2.",
+            },
+        ]
+    elif QuestionType.SPEAKING_1 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are a IELTS program that generates questions for the exams.",
+            },
+            {
+                "role": "user",
+                "content": "The question you have to generate is of type Speaking Task Part 1.",
+            },
+            {
+                "role": "user",
+                "content": "It is mandatory for you to provide your response with the question "
+                           "just with the following json format: {'question': 'question'}",
+            },
+            {
+                "role": "user",
+                "content": "Example output: { 'question': 'Let’s talk about your home town or village. "
+                           "What kind of place is it? What’s the most interesting part of your town/village? "
+                           "What kind of jobs do the people in your town/village do? "
+                           "Would you say it’s a good place to live? (Why?)'}",
+            },
+            {
+                "role": "user",
+                "content": "Generate a question for IELTS exam Speaking Task.",
+            },
+        ]
+    elif QuestionType.SPEAKING_2 == question_type:
+        return [
+            {
+                "role": "user",
+                "content": "You are a IELTS program that generates questions for the exams.",
+            },
+            {
+                "role": "user",
+                "content": "The question you have to generate is of type Speaking Task Part 2.",
+            },
+            {
+                "role": "user",
+                "content": "It is mandatory for you to provide your response with the question "
+                           "just with the following json format: {'question': 'question'}",
+            },
+            {
+                "role": "user",
+                "content": "Example output: { 'question': 'Describe something you own which is very important to you. "
+                           "You should say: where you got it from how long you have had it what you use it for and "
+                           "explain why it is important to you.'}",
+            },
+            {
+                "role": "user",
+                "content": "Generate a question for IELTS exam Speaking Task.",
+            },
+        ]
+    else:
+        raise Exception("Question type not implemented: " + question_type.value)
+
+
+def get_question_tips(question: str, answer: str, correct_answer: str, context: str = None):
+    messages = [
+        {
+            "role": "user",
+            "content": "You are a IELTS exam program that analyzes incorrect answers to questions and gives tips to "
+                       "help students understand why it was a wrong answer and gives helpful insight for the future. "
+                       "The tip should refer to the context and question.",
+        }
+    ]
+
+    if not (context is None or context == ""):
+        messages.append({
+            "role": "user",
+            "content": f"This is the context for the question: {context}",
+        })
+
+    messages.extend([
+        {
+            "role": "user",
+            "content": f"This is the question: {question}",
+        },
+        {
+            "role": "user",
+            "content": f"This is the answer: {answer}",
+        },
+        {
+            "role": "user",
+            "content": f"This is the correct answer: {correct_answer}",
+        }
+    ])
+
+    return messages

helper/constants.py

@@ -0,0 +1,619 @@
+FIREBASE_BUCKET = 'storied-phalanx-349916.appspot.com'
+AUDIO_FILES_PATH = 'download-audio/'
+FIREBASE_LISTENING_AUDIO_FILES_PATH = 'listening_recordings/'
+VIDEO_FILES_PATH = 'download-video/'
+FIREBASE_SPEAKING_VIDEO_FILES_PATH = 'speaking_videos/'
+
+GRADING_TEMPERATURE = 0.1
+TIPS_TEMPERATURE = 0.2
+GEN_QUESTION_TEMPERATURE = 0.7
+GPT_3_5_TURBO = "gpt-3.5-turbo"
+GPT_3_5_TURBO_16K = "gpt-3.5-turbo-16k"
+GPT_3_5_TURBO_INSTRUCT = "gpt-3.5-turbo-instruct"
+
+GRADING_FIELDS = ['comment', 'overall', 'task_response']
+GEN_FIELDS = ['topic']
+GEN_TEXT_FIELDS = ['title']
+LISTENING_GEN_FIELDS = ['transcript', 'exercise']
+READING_EXERCISE_TYPES = ['multipleChoice', 'fillBlanks', 'writeBlanks', 'trueFalse']
+LISTENING_EXERCISE_TYPES = ['multipleChoice', 'writeBlanksQuestions', 'writeBlanksFill', 'writeBlanksForm']
+
+TOTAL_READING_PASSAGE_1_EXERCISES = 13
+TOTAL_READING_PASSAGE_2_EXERCISES = 13
+TOTAL_READING_PASSAGE_3_EXERCISES = 14
+
+TOTAL_LISTENING_SECTION_1_EXERCISES = 10
+TOTAL_LISTENING_SECTION_2_EXERCISES = 10
+TOTAL_LISTENING_SECTION_3_EXERCISES = 10
+TOTAL_LISTENING_SECTION_4_EXERCISES = 10
+
+LISTENING_MIN_TIMER_DEFAULT = 30
+WRITING_MIN_TIMER_DEFAULT = 60
+SPEAKING_MIN_TIMER_DEFAULT = 14
+
+BLACKLISTED_WORDS = ["jesus", "sex", "gay", "lesbian", "homosexual", "god", "angel", "pornography", "beer", "wine",
+                     "cocaine", "drugs", "alcohol", "nudity", "lgbt", "casino", "gambling", "gaming", "catholicism",
+                     "discrimination", "politics", "politic", "christianity", "islam", "christian", "christians",
+                     "jews", "jew", "policies", "human rights", "discrimination", "discriminatory"]
+
+EN_US_VOICES = [
+    {'Gender': 'Female', 'Id': 'Salli', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Salli',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Male', 'Id': 'Matthew', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Matthew',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Female', 'Id': 'Kimberly', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Kimberly',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Female', 'Id': 'Kendra', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Kendra',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Male', 'Id': 'Justin', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Justin',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Male', 'Id': 'Joey', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Joey',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Female', 'Id': 'Joanna', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Joanna',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Female', 'Id': 'Ivy', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Ivy',
+     'SupportedEngines': ['neural', 'standard']}]
+EN_GB_VOICES = [
+    {'Gender': 'Female', 'Id': 'Emma', 'LanguageCode': 'en-GB', 'LanguageName': 'British English', 'Name': 'Emma',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Male', 'Id': 'Brian', 'LanguageCode': 'en-GB', 'LanguageName': 'British English', 'Name': 'Brian',
+     'SupportedEngines': ['neural', 'standard']},
+    {'Gender': 'Female', 'Id': 'Amy', 'LanguageCode': 'en-GB', 'LanguageName': 'British English', 'Name': 'Amy',
+     'SupportedEngines': ['neural', 'standard']}]
+EN_GB_WLS_VOICES = [
+    {'Gender': 'Male', 'Id': 'Geraint', 'LanguageCode': 'en-GB-WLS', 'LanguageName': 'Welsh English', 'Name': 'Geraint',
+     'SupportedEngines': ['standard']}]
+EN_AU_VOICES = [{'Gender': 'Male', 'Id': 'Russell', 'LanguageCode': 'en-AU', 'LanguageName': 'Australian English',
+                 'Name': 'Russell', 'SupportedEngines': ['standard']},
+                {'Gender': 'Female', 'Id': 'Nicole', 'LanguageCode': 'en-AU', 'LanguageName': 'Australian English',
+                 'Name': 'Nicole', 'SupportedEngines': ['standard']}]
+ALL_VOICES = EN_US_VOICES + EN_GB_VOICES + EN_GB_WLS_VOICES + EN_AU_VOICES
+
+NEURAL_EN_US_VOICES = [
+    {'Gender': 'Female', 'Id': 'Danielle', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Danielle',
+     'SupportedEngines': ['neural']},
+    {'Gender': 'Male', 'Id': 'Gregory', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Gregory',
+     'SupportedEngines': ['neural']},
+    {'Gender': 'Male', 'Id': 'Kevin', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Kevin',
+     'SupportedEngines': ['neural']},
+    {'Gender': 'Female', 'Id': 'Ruth', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Ruth',
+     'SupportedEngines': ['neural']},
+    {'Gender': 'Male', 'Id': 'Stephen', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Stephen',
+     'SupportedEngines': ['neural']}]
+NEURAL_EN_GB_VOICES = [
+    {'Gender': 'Male', 'Id': 'Arthur', 'LanguageCode': 'en-GB', 'LanguageName': 'British English', 'Name': 'Arthur',
+     'SupportedEngines': ['neural']}]
+NEURAL_EN_AU_VOICES = [
+    {'Gender': 'Female', 'Id': 'Olivia', 'LanguageCode': 'en-AU', 'LanguageName': 'Australian English',
+     'Name': 'Olivia', 'SupportedEngines': ['neural']}]
+NEURAL_EN_ZA_VOICES = [
+    {'Gender': 'Female', 'Id': 'Ayanda', 'LanguageCode': 'en-ZA', 'LanguageName': 'South African English',
+     'Name': 'Ayanda', 'SupportedEngines': ['neural']}]
+NEURAL_EN_NZ_VOICES = [
+    {'Gender': 'Female', 'Id': 'Aria', 'LanguageCode': 'en-NZ', 'LanguageName': 'New Zealand English', 'Name': 'Aria',
+     'SupportedEngines': ['neural']}]
+NEURAL_EN_IN_VOICES = [
+    {'Gender': 'Female', 'Id': 'Kajal', 'LanguageCode': 'en-IN', 'LanguageName': 'Indian English', 'Name': 'Kajal',
+     'SupportedEngines': ['neural']}]
+NEURAL_EN_IE_VOICES = [
+    {'Gender': 'Female', 'Id': 'Niamh', 'LanguageCode': 'en-IE', 'LanguageName': 'Irish English', 'Name': 'Niamh',
+     'SupportedEngines': ['neural']}]
+ALL_NEURAL_VOICES = NEURAL_EN_US_VOICES + NEURAL_EN_GB_VOICES + NEURAL_EN_AU_VOICES + NEURAL_EN_ZA_VOICES + NEURAL_EN_NZ_VOICES + NEURAL_EN_IE_VOICES
+
+MALE_VOICES = [item for item in ALL_VOICES if item.get('Gender') == 'Male']
+FEMALE_VOICES = [item for item in ALL_VOICES if item.get('Gender') == 'Female']
+
+MALE_NEURAL_VOICES = [item for item in ALL_NEURAL_VOICES if item.get('Gender') == 'Male']
+FEMALE_NEURAL_VOICES = [item for item in ALL_NEURAL_VOICES if item.get('Gender') == 'Female']
+
+difficulties = ["easy", "medium", "hard"]
+
+topics = [
+    "Art and Creativity",
+    "History of Ancient Civilizations",
+    "Environmental Conservation",
+    "Space Exploration",
+    "Artificial Intelligence",
+    "Climate Change",
+    "World Religions",
+    "The Human Brain",
+    "Renewable Energy",
+    "Cultural Diversity",
+    "Modern Technology Trends",
+    "Sustainable Agriculture",
+    "Globalization",
+    "Natural Disasters",
+    "Cybersecurity",
+    "Philosophy of Ethics",
+    "Robotics",
+    "Health and Wellness",
+    "Literature and Classics",
+    "World Geography",
+    "Music and Its Influence",
+    "Social Media Impact",
+    "Food Sustainability",
+    "Economics and Markets",
+    "Human Evolution",
+    "Political Systems",
+    "Mental Health Awareness",
+    "Quantum Physics",
+    "Biodiversity",
+    "Education Reform",
+    "Animal Rights",
+    "The Industrial Revolution",
+    "Future of Work",
+    "Film and Cinema",
+    "Genetic Engineering",
+    "Climate Policy",
+    "Space Travel",
+    "Renewable Energy Sources",
+    "Cultural Heritage Preservation",
+    "Modern Art Movements",
+    "Immigration Issues",
+    "Sustainable Transportation",
+    "The History of Medicine",
+    "Artificial Neural Networks",
+    "Climate Adaptation",
+    "Philosophy of Existence",
+    "Augmented Reality",
+    "Yoga and Meditation",
+    "Literary Genres",
+    "World Oceans",
+    "Social Networking",
+    "Sustainable Fashion",
+    "International Trade",
+    "Prehistoric Era",
+    "Democracy and Governance",
+    "Postcolonial Literature",
+    "Geopolitics",
+    "Psychology and Behavior",
+    "Nanotechnology",
+    "Endangered Species",
+    "Education Technology",
+    "Renaissance Art",
+    "Renewable Energy Policy",
+    "Cultural Festivals",
+    "Modern Architecture",
+    "Climate Resilience",
+    "Artificial Life",
+    "Fitness and Nutrition",
+    "Classic Literature Adaptations",
+    "World History Wars",
+    "Ethical Dilemmas",
+    "Internet of Things (IoT)",
+    "Meditation Practices",
+    "Literary Symbolism",
+    "Marine Conservation",
+    "Social Justice Movements",
+    "Sustainable Tourism",
+    "International Finance",
+    "Ancient Philosophy",
+    "Cold War Era",
+    "Behavioral Economics",
+    "Space Colonization",
+    "Clean Energy Initiatives",
+    "Cultural Exchange",
+    "Modern Sculpture",
+    "Climate Mitigation",
+    "Artificial Intelligence Ethics",
+    "Mindfulness",
+    "Literary Criticism",
+    "Wildlife Conservation",
+    "Political Activism",
+    "Renewable Energy Innovations",
+    "History of Mathematics",
+    "Human-Computer Interaction",
+    "Global Health",
+    "Cultural Appropriation",
+    "Traditional cuisine and culinary arts",
+    "Local music and dance traditions",
+    "Cultural festivals and celebrations",
+    "History of the region and historical landmarks",
+    "Traditional crafts and artisanal skills",
+    "Wildlife and conservation efforts",
+    "Local sports and athletic competitions",
+    "Fashion trends and clothing styles",
+    "Literature and poetry from the region",
+    "Education systems and advancements",
+    "Healthcare services and medical innovations",
+    "Family values and social dynamics",
+    "Travel destinations and tourist attractions",
+    "Environmental sustainability projects",
+    "Technological developments and innovations",
+    "Entrepreneurship and business ventures",
+    "Youth empowerment initiatives",
+    "Art exhibitions and cultural events",
+    "Philanthropy and community development projects",
+    "Political developments and civic engagement efforts"
+]
+
+two_people_scenarios = [
+    "Booking a table at a restaurant",
+    "Making a doctor's appointment",
+    "Asking for directions to a tourist attraction",
+    "Inquiring about public transportation options",
+    "Discussing weekend plans with a friend",
+    "Ordering food at a café",
+    "Renting a bicycle for a day",
+    "Arranging a meeting with a colleague",
+    "Talking to a real estate agent about renting an apartment",
+    "Discussing travel plans for an upcoming vacation",
+    "Checking the availability of a hotel room",
+    "Talking to a car rental service",
+    "Asking for recommendations at a library",
+    "Inquiring about opening hours at a museum",
+    "Discussing the weather forecast",
+    "Shopping for groceries",
+    "Renting a movie from a video store",
+    "Booking a flight ticket",
+    "Discussing a school assignment with a classmate",
+    "Making a reservation for a spa appointment",
+    "Talking to a customer service representative about a product issue",
+    "Discussing household chores with a family member",
+    "Planning a surprise party for a friend",
+    "Talking to a coworker about a project deadline",
+    "Inquiring about a gym membership",
+    "Discussing the menu options at a fast-food restaurant",
+    "Talking to a neighbor about a community event",
+    "Asking for help with computer problems",
+    "Discussing a recent sports game with a sports enthusiast",
+    "Talking to a pet store employee about buying a pet",
+    "Asking for information about a local farmer's market",
+    "Discussing the details of a home renovation project",
+    "Talking to a coworker about office supplies",
+    "Making plans for a family picnic",
+    "Inquiring about admission requirements at a university",
+    "Discussing the features of a new smartphone with a salesperson",
+    "Talking to a mechanic about car repairs",
+    "Making arrangements for a child's birthday party",
+    "Discussing a new diet plan with a nutritionist",
+    "Asking for information about a music concert",
+    "Talking to a hairdresser about getting a haircut",
+    "Inquiring about a language course at a language school",
+    "Discussing plans for a weekend camping trip",
+    "Talking to a bank teller about opening a new account",
+    "Ordering a drink at a coffee shop",
+    "Discussing a new book with a book club member",
+    "Talking to a librarian about library services",
+    "Asking for advice on finding a job",
+    "Discussing plans for a garden makeover with a landscaper",
+    "Talking to a travel agent about a cruise vacation",
+    "Inquiring about a fitness class at a gym",
+    "Ordering flowers for a special occasion",
+    "Discussing a new exercise routine with a personal trainer",
+    "Talking to a teacher about a child's progress in school",
+    "Asking for information about a local art exhibition",
+    "Discussing a home improvement project with a contractor",
+    "Talking to a babysitter about childcare arrangements",
+    "Making arrangements for a car service appointment",
+    "Inquiring about a photography workshop at a studio",
+    "Discussing plans for a family reunion with a relative",
+    "Talking to a tech support representative about computer issues",
+    "Asking for recommendations on pet grooming services",
+    "Discussing weekend plans with a significant other",
+    "Talking to a counselor about personal issues",
+    "Inquiring about a music lesson with a music teacher",
+    "Ordering a pizza for delivery",
+    "Making a reservation for a taxi",
+    "Discussing a new recipe with a chef",
+    "Talking to a fitness trainer about weight loss goals",
+    "Inquiring about a dance class at a dance studio",
+    "Ordering a meal at a food truck",
+    "Discussing plans for a weekend getaway with a partner",
+    "Talking to a florist about wedding flower arrangements",
+    "Asking for advice on home decorating",
+    "Discussing plans for a charity fundraiser event",
+    "Talking to a pet sitter about taking care of pets",
+    "Making arrangements for a spa day with a friend",
+    "Asking for recommendations on home improvement stores",
+    "Discussing weekend plans with a travel enthusiast",
+    "Talking to a car mechanic about car maintenance",
+    "Inquiring about a cooking class at a culinary school",
+    "Ordering a sandwich at a deli",
+    "Discussing plans for a family holiday party",
+    "Talking to a personal assistant about organizing tasks",
+    "Asking for information about a local theater production",
+    "Discussing a new DIY project with a home improvement expert",
+    "Talking to a wine expert about wine pairing",
+    "Making arrangements for a pet adoption",
+    "Asking for advice on planning a wedding"
+]
+
+social_monologue_contexts = [
+    "A guided tour of a historical museum",
+    "An introduction to a new city for tourists",
+    "An orientation session for new university students",
+    "A safety briefing for airline passengers",
+    "An explanation of the process of recycling",
+    "A lecture on the benefits of a healthy diet",
+    "A talk on the importance of time management",
+    "A monologue about wildlife conservation",
+    "An overview of local public transportation options",
+    "A presentation on the history of cinema",
+    "An introduction to the art of photography",
+    "A discussion about the effects of climate change",
+    "An overview of different types of cuisine",
+    "A lecture on the principles of financial planning",
+    "A monologue about sustainable energy sources",
+    "An explanation of the process of online shopping",
+    "A guided tour of a botanical garden",
+    "An introduction to a local wildlife sanctuary",
+    "A safety briefing for hikers in a national park",
+    "A talk on the benefits of physical exercise",
+    "A lecture on the principles of effective communication",
+    "A monologue about the impact of social media",
+    "An overview of the history of a famous landmark",
+    "An introduction to the world of fashion design",
+    "A discussion about the challenges of global poverty",
+    "An explanation of the process of organic farming",
+    "A presentation on the history of space exploration",
+    "An overview of traditional music from different cultures",
+    "A lecture on the principles of effective leadership",
+    "A monologue about the influence of technology",
+    "A guided tour of a famous archaeological site",
+    "An introduction to a local wildlife rehabilitation center",
+    "A safety briefing for visitors to a science museum",
+    "A talk on the benefits of learning a new language",
+    "A lecture on the principles of architectural design",
+    "A monologue about the impact of renewable energy",
+    "An explanation of the process of online banking",
+    "A presentation on the history of a famous art movement",
+    "An overview of traditional clothing from various regions",
+    "A lecture on the principles of sustainable agriculture",
+    "A discussion about the challenges of urban development",
+    "A monologue about the influence of social norms",
+    "A guided tour of a historical battlefield",
+    "An introduction to a local animal shelter",
+    "A safety briefing for participants in a charity run",
+    "A talk on the benefits of community involvement",
+    "A lecture on the principles of sustainable tourism",
+    "A monologue about the impact of alternative medicine",
+    "An explanation of the process of wildlife tracking",
+    "A presentation on the history of a famous inventor",
+    "An overview of traditional dance forms from different cultures",
+    "A lecture on the principles of ethical business practices",
+    "A discussion about the challenges of healthcare access",
+    "A monologue about the influence of cultural traditions",
+    "A guided tour of a famous lighthouse",
+    "An introduction to a local astronomy observatory",
+    "A safety briefing for participants in a team-building event",
+    "A talk on the benefits of volunteering",
+    "A lecture on the principles of wildlife protection",
+    "A monologue about the impact of space exploration",
+    "An explanation of the process of wildlife photography",
+    "A presentation on the history of a famous musician",
+    "An overview of traditional art forms from different cultures",
+    "A lecture on the principles of effective education",
+    "A discussion about the challenges of sustainable development",
+    "A monologue about the influence of cultural diversity",
+    "A guided tour of a famous national park",
+    "An introduction to a local marine conservation project",
+    "A safety briefing for participants in a hot air balloon ride",
+    "A talk on the benefits of cultural exchange programs",
+    "A lecture on the principles of wildlife conservation",
+    "A monologue about the impact of technological advancements",
+    "An explanation of the process of wildlife rehabilitation",
+    "A presentation on the history of a famous explorer",
+    "An overview of traditional storytelling from different cultures",
+    "A lecture on the principles of effective marketing",
+    "A discussion about the challenges of environmental sustainability",
+    "A monologue about the influence of social entrepreneurship",
+    "A guided tour of a famous historical estate",
+    "An introduction to a local marine life research center",
+    "A safety briefing for participants in a zip-lining adventure",
+    "A talk on the benefits of cultural preservation",
+    "A lecture on the principles of wildlife ecology",
+    "A monologue about the impact of space technology",
+    "An explanation of the process of wildlife conservation",
+    "A presentation on the history of a famous scientist",
+    "An overview of traditional crafts and artisans from different cultures",
+    "A lecture on the principles of effective intercultural communication"
+]
+
+four_people_scenarios = [
+    "A university lecture on history",
+    "A physics class discussing Newton's laws",
+    "A medical school seminar on anatomy",
+    "A training session on computer programming",
+    "A business school lecture on marketing strategies",
+    "A chemistry lab experiment and discussion",
+    "A language class practicing conversational skills",
+    "A workshop on creative writing techniques",
+    "A high school math lesson on calculus",
+    "A training program for customer service representatives",
+    "A lecture on environmental science and sustainability",
+    "A psychology class exploring human behavior",
+    "A music theory class analyzing compositions",
+    "A nursing school simulation for patient care",
+    "A computer science class on algorithms",
+    "A workshop on graphic design principles",
+    "A law school lecture on constitutional law",
+    "A geology class studying rock formations",
+    "A vocational training program for electricians",
+    "A history seminar focusing on ancient civilizations",
+    "A biology class dissecting specimens",
+    "A financial literacy course for adults",
+    "A literature class discussing classic novels",
+    "A training session for emergency response teams",
+    "A sociology lecture on social inequality",
+    "An art class exploring different painting techniques",
+    "A medical school seminar on diagnosis",
+    "A programming bootcamp teaching web development",
+    "An economics class analyzing market trends",
+    "A chemistry lab experiment on chemical reactions",
+    "A language class practicing pronunciation",
+    "A workshop on public speaking skills",
+    "A high school physics lesson on electromagnetism",
+    "A training program for IT professionals",
+    "A lecture on climate change and its effects",
+    "A psychology class studying cognitive psychology",
+    "A music class composing original songs",
+    "A nursing school simulation for patient assessment",
+    "A computer science class on data structures",
+    "A workshop on 3D modeling and animation",
+    "A law school lecture on contract law",
+    "A geography class examining world maps",
+    "A vocational training program for plumbers",
+    "A history seminar discussing revolutions",
+    "A biology class exploring genetics",
+    "A financial literacy course for teens",
+    "A literature class analyzing poetry",
+    "A training session for public speaking coaches",
+    "A sociology lecture on cultural diversity",
+    "An art class creating sculptures",
+    "A medical school seminar on surgical techniques",
+    "A programming bootcamp teaching app development",
+    "An economics class on global trade policies",
+    "A chemistry lab experiment on chemical bonding",
+    "A language class discussing idiomatic expressions",
+    "A workshop on conflict resolution",
+    "A high school biology lesson on evolution",
+    "A training program for project managers",
+    "A lecture on renewable energy sources",
+    "A psychology class on abnormal psychology",
+    "A music class rehearsing for a performance",
+    "A nursing school simulation for emergency response",
+    "A computer science class on cybersecurity",
+    "A workshop on digital marketing strategies",
+    "A law school lecture on intellectual property",
+    "A geology class analyzing seismic activity",
+    "A vocational training program for carpenters",
+    "A history seminar on the Renaissance",
+    "A chemistry class synthesizing compounds",
+    "A financial literacy course for seniors",
+    "A literature class interpreting Shakespearean plays",
+    "A training session for negotiation skills",
+    "A sociology lecture on urbanization",
+    "An art class creating digital art",
+    "A medical school seminar on patient communication",
+    "A programming bootcamp teaching mobile app development",
+    "An economics class on fiscal policy",
+    "A physics lab experiment on electromagnetism",
+    "A language class on cultural immersion",
+    "A workshop on time management",
+    "A high school chemistry lesson on stoichiometry",
+    "A training program for HR professionals",
+    "A lecture on space exploration and astronomy",
+    "A psychology class on human development",
+    "A music class practicing for a recital",
+    "A nursing school simulation for triage",
+    "A computer science class on web development frameworks",
+    "A workshop on team-building exercises",
+    "A law school lecture on criminal law",
+    "A geography class studying world cultures",
+    "A vocational training program for HVAC technicians",
+    "A history seminar on ancient civilizations",
+    "A biology class examining ecosystems",
+    "A financial literacy course for entrepreneurs",
+    "A literature class analyzing modern literature",
+    "A training session for leadership skills",
+    "A sociology lecture on gender studies",
+    "An art class exploring multimedia art",
+    "A medical school seminar on patient diagnosis",
+    "A programming bootcamp teaching software architecture"
+]
+
+academic_subjects = [
+    "Astrophysics",
+    "Microbiology",
+    "Political Science",
+    "Environmental Science",
+    "Literature",
+    "Biochemistry",
+    "Sociology",
+    "Art History",
+    "Geology",
+    "Economics",
+    "Psychology",
+    "History of Architecture",
+    "Linguistics",
+    "Neurobiology",
+    "Anthropology",
+    "Quantum Mechanics",
+    "Urban Planning",
+    "Philosophy",
+    "Marine Biology",
+    "International Relations",
+    "Medieval History",
+    "Geophysics",
+    "Finance",
+    "Educational Psychology",
+    "Graphic Design",
+    "Paleontology",
+    "Macroeconomics",
+    "Cognitive Psychology",
+    "Renaissance Art",
+    "Archaeology",
+    "Microeconomics",
+    "Social Psychology",
+    "Contemporary Art",
+    "Meteorology",
+    "Political Philosophy",
+    "Space Exploration",
+    "Cognitive Science",
+    "Classical Music",
+    "Oceanography",
+    "Public Health",
+    "Gender Studies",
+    "Baroque Art",
+    "Volcanology",
+    "Business Ethics",
+    "Music Composition",
+    "Environmental Policy",
+    "Media Studies",
+    "Ancient History",
+    "Seismology",
+    "Marketing",
+    "Human Development",
+    "Modern Art",
+    "Astronomy",
+    "International Law",
+    "Developmental Psychology",
+    "Film Studies",
+    "American History",
+    "Soil Science",
+    "Entrepreneurship",
+    "Clinical Psychology",
+    "Contemporary Dance",
+    "Space Physics",
+    "Political Economy",
+    "Cognitive Neuroscience",
+    "20th Century Literature",
+    "Public Administration",
+    "European History",
+    "Atmospheric Science",
+    "Supply Chain Management",
+    "Social Work",
+    "Japanese Literature",
+    "Planetary Science",
+    "Labor Economics",
+    "Industrial-Organizational Psychology",
+    "French Philosophy",
+    "Biogeochemistry",
+    "Strategic Management",
+    "Educational Sociology",
+    "Postmodern Literature",
+    "Public Relations",
+    "Middle Eastern History",
+    "Oceanography",
+    "International Development",
+    "Human Resources Management",
+    "Educational Leadership",
+    "Russian Literature",
+    "Quantum Chemistry",
+    "Environmental Economics",
+    "Environmental Psychology",
+    "Ancient Philosophy",
+    "Immunology",
+    "Comparative Politics",
+    "Child Development",
+    "Fashion Design",
+    "Geological Engineering",
+    "Macroeconomic Policy",
+    "Media Psychology",
+    "Byzantine Art",
+    "Ecology",
+    "International Business"
+]
+
+

helper/exercises.py

@@ -0,0 +1,769 @@
+import queue
+import nltk
+import random
+import re
+import uuid
+
+from helper.api_messages import QuestionType
+from helper.firebase_helper import get_all
+from helper.openai_interface import make_openai_instruct_call
+from helper.token_counter import count_tokens
+from helper.constants import *
+from wonderwords import RandomWord
+
+nltk.download('words')
+
+
+def divide_number_into_parts(number, parts):
+    if number < parts:
+        return None
+
+    part_size = number // parts
+    remaining = number % parts
+
+    q = queue.Queue()
+
+    for i in range(parts):
+        if i < remaining:
+            q.put(part_size + 1)
+        else:
+            q.put(part_size)
+
+    return q
+
+
+def fix_exercise_ids(exercise, start_id):
+    # Initialize the starting ID for the first exercise
+    current_id = start_id
+
+    questions = exercise["questions"]
+
+    # Iterate through questions and update the "id" value
+    for question in questions:
+        question["id"] = str(current_id)
+        current_id += 1
+
+    return exercise
+
+
+def replace_first_occurrences_with_placeholders(text: str, words_to_replace: list, start_id):
+    for i, word in enumerate(words_to_replace, start=start_id):
+        # Create a case-insensitive regular expression pattern
+        pattern = re.compile(r'\b' + re.escape(word) + r'\b', re.IGNORECASE)
+        placeholder = '{{' + str(i) + '}}'
+        text = pattern.sub(placeholder, text, 1)
+    return text
+
+
+def replace_first_occurrences_with_placeholders_notes(notes: list, words_to_replace: list, start_id):
+    replaced_notes = []
+    for i, note in enumerate(notes, start=0):
+        word = words_to_replace[i]
+        pattern = re.compile(r'\b' + re.escape(word) + r'\b', re.IGNORECASE)
+        placeholder = '{{' + str(start_id + i) + '}}'
+        note = pattern.sub(placeholder, note, 1)
+        replaced_notes.append(note)
+    return replaced_notes
+
+
+def add_random_words_and_shuffle(word_array, num_random_words):
+    r = RandomWord()
+    random_words_selected = r.random_words(num_random_words)
+
+    combined_array = word_array + random_words_selected
+
+    random.shuffle(combined_array)
+
+    return combined_array
+
+
+def fillblanks_build_solutions_array(words, start_id):
+    solutions = []
+    for i, word in enumerate(words, start=start_id):
+        solutions.append(
+            {
+                "id": str(i),
+                "solution": word
+            }
+        )
+    return solutions
+
+
+def remove_excess_questions(questions: [], quantity):
+    count_true = 0
+    result = []
+
+    for item in reversed(questions):
+        if item.get('solution') == 'true' and count_true < quantity:
+            count_true += 1
+        else:
+            result.append(item)
+
+    result.reverse()
+    return result
+
+
+def build_write_blanks_text(questions: [], start_id):
+    result = ""
+    for i, q in enumerate(questions, start=start_id):
+        placeholder = '{{' + str(i) + '}}'
+        result = result + q["question"] + placeholder + "\\n"
+    return result
+
+
+def build_write_blanks_text_form(form: [], start_id):
+    result = ""
+    replaced_words = []
+    for i, entry in enumerate(form, start=start_id):
+        placeholder = '{{' + str(i) + '}}'
+        # Use regular expression to find the string after ':'
+        match = re.search(r'(?<=:)\s*(.*)', entry)
+        # Extract the matched string
+        original_string = match.group(1)
+        # Split the string into words
+        words = re.findall(r'\b\w+\b', original_string)
+        # Remove words with only one letter
+        filtered_words = [word for word in words if len(word) > 1]
+        # Choose a random word from the list of words
+        selected_word = random.choice(filtered_words)
+        pattern = re.compile(r'\b' + re.escape(selected_word) + r'\b', re.IGNORECASE)
+
+        # Replace the chosen word with the placeholder
+        replaced_string = pattern.sub(placeholder, original_string, 1)
+        # Construct the final replaced string
+        replaced_string = entry.replace(original_string, replaced_string)
+
+        result = result + replaced_string + "\\n"
+        # Save the replaced word or use it as needed
+        # For example, you can save it to a file or a list
+        replaced_words.append(selected_word)
+    return result, replaced_words
+
+
+def build_write_blanks_solutions(questions: [], start_id):
+    solutions = []
+    for i, q in enumerate(questions, start=start_id):
+        solution = [q["possible_answers"]] if isinstance(q["possible_answers"], str) else q["possible_answers"]
+
+        solutions.append(
+            {
+                "id": str(i),
+                "solution": solution
+            }
+        )
+    return solutions
+
+
+def build_write_blanks_solutions_listening(words: [], start_id):
+    solutions = []
+    for i, word in enumerate(words, start=start_id):
+        solution = [word] if isinstance(word, str) else word
+
+        solutions.append(
+            {
+                "id": str(i),
+                "solution": solution
+            }
+        )
+    return solutions
+
+
+def generate_reading_passage(type: QuestionType, topic: str):
+    gen_reading_passage_1 = "Generate an extensive text for IELTS " + type.value + ", of at least 1500 words, on the topic " \
+                                                                                   "of '" + topic + "'. The passage should offer a substantial amount of " \
+                                                                                                    "information, analysis, or narrative " \
+                                                                                                    "relevant to the chosen subject matter. This text passage aims to serve as the primary reading " \
+                                                                                                    "section of an IELTS test, providing an in-depth and comprehensive exploration of the topic. " \
+                                                                                                    "Make sure that the generated text does not contain forbidden subjects in muslim countries." \
+                                                                                                    "Provide your response in this json format: {\"title\": \"title of the text\", \"text\": \"generated text\"}"
+    token_count = count_tokens(gen_reading_passage_1)["n_tokens"]
+    return make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_reading_passage_1, token_count, GEN_TEXT_FIELDS,
+                                     GEN_QUESTION_TEMPERATURE)
+
+
+def generate_listening_1_conversation(topic: str):
+    gen_listening_1_conversation_2_people = "Compose an authentic conversation between two individuals in the everyday " \
+                                            "social context of '" + topic + "'. Please include random names and genders " \
+                                                                            "for the characters in your dialogue. " \
+                                                                            "Make sure that the generated conversation does not contain forbidden subjects in muslim countries."
+    token_count = count_tokens(gen_listening_1_conversation_2_people)["n_tokens"]
+    response = make_openai_instruct_call(
+        GPT_3_5_TURBO_INSTRUCT,
+        gen_listening_1_conversation_2_people,
+        token_count,
+        None,
+        GEN_QUESTION_TEMPERATURE
+    )
+
+    conversation_json = '{"conversation": [{"name": "name", "gender": "gender", "text": "text"}]}'
+
+    parse_conversation = "Parse this conversation: '" + response + "' to the following json format: " + conversation_json
+
+    token_count = count_tokens(parse_conversation)["n_tokens"]
+    processed = make_openai_instruct_call(
+        GPT_3_5_TURBO_INSTRUCT,
+        parse_conversation,
+        token_count,
+        ['conversation'],
+        GEN_QUESTION_TEMPERATURE
+    )
+
+    name_to_voice = {}
+    for segment in processed['conversation']:
+        if 'voice' not in segment:
+            name = segment['name']
+            if name in name_to_voice:
+                voice = name_to_voice[name]
+            else:
+                if segment['gender'].lower() == 'male':
+                    voice = random.choice(MALE_NEURAL_VOICES)['Id']
+                else:
+                    voice = random.choice(FEMALE_NEURAL_VOICES)['Id']
+                name_to_voice[name] = voice
+            segment['voice'] = voice
+    return response, processed
+
+
+def generate_listening_2_monologue(topic: str):
+    gen_listening_2_monologue_social = "Generate a comprehensive monologue set in the social context of: '" + topic + "'. Make sure that the generated monologue does not contain forbidden subjects in muslim countries."
+    token_count = count_tokens(gen_listening_2_monologue_social)["n_tokens"]
+    response = make_openai_instruct_call(
+        GPT_3_5_TURBO_INSTRUCT,
+        gen_listening_2_monologue_social,
+        token_count,
+        None,
+        GEN_QUESTION_TEMPERATURE
+    )
+    return response
+
+
+def generate_listening_3_conversation(topic: str):
+    gen_listening_3_conversation_4_people = "Compose an authentic and elaborate conversation between up to four individuals " \
+                                            "in the everyday social context of '" + topic + \
+                                            "'. Please include random names and genders for the characters in your dialogue. " \
+                                            "Make sure that the generated conversation does not contain forbidden subjects in muslim countries."
+    token_count = count_tokens(gen_listening_3_conversation_4_people)["n_tokens"]
+    response = make_openai_instruct_call(
+        GPT_3_5_TURBO_INSTRUCT,
+        gen_listening_3_conversation_4_people,
+        token_count,
+        None,
+        GEN_QUESTION_TEMPERATURE
+    )
+    conversation_json = '{"conversation": [{"name": "name", "gender": "gender", "text": "text"}]}'
+
+    parse_conversation = "Parse this conversation: '" + response + "' to the following json format: " + conversation_json
+
+    token_count = count_tokens(parse_conversation)["n_tokens"]
+    processed = make_openai_instruct_call(
+        GPT_3_5_TURBO_INSTRUCT,
+        parse_conversation,
+        token_count,
+        ['conversation'],
+        GEN_QUESTION_TEMPERATURE
+    )
+
+    name_to_voice = {}
+    for segment in processed['conversation']:
+        if 'voice' not in segment:
+            name = segment['name']
+            if name in name_to_voice:
+                voice = name_to_voice[name]
+            else:
+                if segment['gender'].lower() == 'male':
+                    voice = random.choice(MALE_NEURAL_VOICES)['Id']
+                else:
+                    voice = random.choice(FEMALE_NEURAL_VOICES)['Id']
+                name_to_voice[name] = voice
+            segment['voice'] = voice
+    return response, processed
+
+
+def generate_listening_4_monologue(topic: str):
+    gen_listening_4_monologue_academic = "Generate a comprehensive monologue an academic subject of: '" + topic + "'. Make sure that the generated monologue does not contain forbidden subjects in muslim countries."
+    token_count = count_tokens(gen_listening_4_monologue_academic)["n_tokens"]
+    response = make_openai_instruct_call(
+        GPT_3_5_TURBO_INSTRUCT,
+        gen_listening_4_monologue_academic,
+        token_count,
+        None,
+        GEN_QUESTION_TEMPERATURE
+    )
+    return response
+
+
+def generate_reading_exercises(passage: str, req_exercises: list, number_of_exercises_q, start_id, difficulty):
+    exercises = []
+    for req_exercise in req_exercises:
+        number_of_exercises = number_of_exercises_q.get()
+
+        if req_exercise == "multipleChoice":
+            question = gen_multiple_choice_exercise(passage, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added multiple choice: " + str(question))
+        elif req_exercise == "fillBlanks":
+            question = gen_summary_fill_blanks_exercise(passage, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added fill blanks: " + str(question))
+        elif req_exercise == "trueFalse":
+            question = gen_true_false_not_given_exercise(passage, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added trueFalse: " + str(question))
+        elif req_exercise == "writeBlanks":
+            question = gen_write_blanks_exercise(passage, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added write blanks: " + str(question))
+
+        start_id = start_id + number_of_exercises
+
+    return exercises
+
+
+def generate_listening_conversation_exercises(conversation: str, req_exercises: list, number_of_exercises_q, start_id, difficulty):
+    exercises = []
+    for req_exercise in req_exercises:
+        number_of_exercises = number_of_exercises_q.get()
+
+        if req_exercise == "multipleChoice":
+            question = gen_multiple_choice_exercise_listening_conversation(conversation, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added multiple choice: " + str(question))
+        elif req_exercise == "writeBlanksQuestions":
+            question = gen_write_blanks_questions_exercise_listening_conversation(conversation, number_of_exercises,
+                                                                                  start_id, difficulty)
+            exercises.append(question)
+            print("Added write blanks questions: " + str(question))
+        elif req_exercise == "writeBlanksFill":
+            question = gen_write_blanks_notes_exercise_listening_conversation(conversation, number_of_exercises,
+                                                                              start_id, difficulty)
+            exercises.append(question)
+            print("Added write blanks notes: " + str(question))
+        elif req_exercise == "writeBlanksForm":
+            question = gen_write_blanks_form_exercise_listening_conversation(conversation, number_of_exercises,
+                                                                             start_id, difficulty)
+            exercises.append(question)
+            print("Added write blanks form: " + str(question))
+
+        start_id = start_id + number_of_exercises
+
+    return exercises
+
+
+def generate_listening_monologue_exercises(monologue: str, req_exercises: list, number_of_exercises_q, start_id, difficulty):
+    exercises = []
+    for req_exercise in req_exercises:
+        number_of_exercises = number_of_exercises_q.get()
+
+        if req_exercise == "multipleChoice":
+            question = gen_multiple_choice_exercise_listening_monologue(monologue, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added multiple choice: " + str(question))
+        elif req_exercise == "writeBlanksQuestions":
+            question = gen_write_blanks_questions_exercise_listening_monologue(monologue, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added write blanks questions: " + str(question))
+        elif req_exercise == "writeBlanksFill":
+            question = gen_write_blanks_notes_exercise_listening_monologue(monologue, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added write blanks notes: " + str(question))
+        elif req_exercise == "writeBlanksForm":
+            question = gen_write_blanks_form_exercise_listening_monologue(monologue, number_of_exercises, start_id, difficulty)
+            exercises.append(question)
+            print("Added write blanks form: " + str(question))
+
+        start_id = start_id + number_of_exercises
+
+    return exercises
+
+
+def gen_multiple_choice_exercise(text: str, quantity: int, start_id, difficulty):
+    gen_multiple_choice_for_text = "Generate " + str(quantity) + " " + difficulty + " difficulty multiple choice questions for this text: " \
+                                                                 "'" + text + "'\n" \
+                                                                              "Use this format: \"questions\": [{\"id\": \"9\", \"options\": [{\"id\": \"A\", \"text\": " \
+                                                                              "\"Economic benefits\"}, {\"id\": \"B\", \"text\": \"Government regulations\"}, {\"id\": \"C\", \"text\": " \
+                                                                              "\"Concerns about climate change\"}, {\"id\": \"D\", \"text\": \"Technological advancement\"}], " \
+                                                                              "\"prompt\": \"What is the main reason for the shift towards renewable energy sources?\", " \
+                                                                              "\"solution\": \"C\", \"variant\": \"text\"}]"
+    token_count = count_tokens(gen_multiple_choice_for_text)["n_tokens"]
+    mc_questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_multiple_choice_for_text, token_count,
+                                             None,
+                                             GEN_QUESTION_TEMPERATURE)
+    parse_mc_questions = "Parse the questions into this json format: {\"questions\": [{\"id\": \"9\", \"options\": [{\"id\": \"A\", \"text\": " \
+                         "\"Economic benefits\"}, {\"id\": \"B\", \"text\": \"Government regulations\"}, {\"id\": \"C\", \"text\": " \
+                         "\"Concerns about climate change\"}, {\"id\": \"D\", \"text\": \"Technological advancement\"}], " \
+                         "\"prompt\": \"What is the main reason for the shift towards renewable energy sources?\", " \
+                         "\"solution\": \"C\", \"variant\": \"text\"}]}. \nThe questions: '" + mc_questions + "'"
+    token_count = count_tokens(parse_mc_questions)["n_tokens"]
+    question = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count,
+                                         ["questions"],
+                                         GEN_QUESTION_TEMPERATURE)
+    return {
+        "id": str(uuid.uuid4()),
+        "prompt": "Select the appropriate option.",
+        "questions": fix_exercise_ids(question, start_id)["questions"],
+        "type": "multipleChoice",
+    }
+
+
+def gen_summary_fill_blanks_exercise(text: str, quantity: int, start_id, difficulty):
+    gen_summary_for_text = "Summarize this text: " + text
+    token_count = count_tokens(gen_summary_for_text)["n_tokens"]
+    text_summary = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_summary_for_text, token_count,
+                                             None,
+                                             GEN_QUESTION_TEMPERATURE)
+
+    gen_words_to_replace = "Select " + str(
+        quantity) + " " + difficulty + " difficulty words, it must be words and not expressions, from the summary and respond in this " \
+                    "JSON format: { \"words\": [\"word_1\", \"word_2\"] }. The summary is: " + text_summary
+    token_count = count_tokens(gen_words_to_replace)["n_tokens"]
+    words_to_replace = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_words_to_replace, token_count,
+                                                 ["words"],
+                                                 GEN_QUESTION_TEMPERATURE)["words"]
+
+    replaced_summary = replace_first_occurrences_with_placeholders(text_summary, words_to_replace, start_id)
+    options_words = add_random_words_and_shuffle(words_to_replace, 5)
+    solutions = fillblanks_build_solutions_array(words_to_replace, start_id)
+
+    return {
+        "allowRepetition": True,
+        "id": str(uuid.uuid4()),
+        "prompt": "Complete the summary below. Click a blank to select the corresponding word(s) for it.\\nThere are "
+                  "more words than spaces so you will not use them all. You may use any of the words more than once.",
+        "solutions": solutions,
+        "text": replaced_summary,
+        "type": "fillBlanks",
+        "words": options_words
+
+    }
+
+
+def gen_true_false_not_given_exercise(text: str, quantity: int, start_id, difficulty):
+    gen_true_false_not_given = "Generate " + str(
+        quantity) + " " + difficulty + " difficulty statements in JSON format (True, False, or Not Given) " \
+                    "based on the provided text. Ensure that your statements " \
+                    "accurately represent information or inferences from the " \
+                    "text, and provide a variety of responses, including, at least one of each True, " \
+                    "False, and Not Given, as appropriate, in the JSON structure " \
+                    "{\"prompts\":[{\"prompt\": \"statement_1\", \"solution\": " \
+                    "\"true/false/not_given\"}, {\"prompt\": \"statement_2\", " \
+                    "\"solution\": \"true/false/not_given\"}]}. Reference text: " + text
+
+    token_count = count_tokens(gen_true_false_not_given)["n_tokens"]
+    questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_true_false_not_given, token_count,
+                                          ["prompts"],
+                                          GEN_QUESTION_TEMPERATURE)["prompts"]
+    if len(questions) > quantity:
+        questions = remove_excess_questions(questions, len(questions) - quantity)
+
+    for i, question in enumerate(questions, start=start_id):
+        question["id"] = str(i)
+
+    return {
+        "id": str(uuid.uuid4()),
+        "prompt": "Do the following statements agree with the information given in the Reading Passage?",
+        "questions": questions,
+        "type": "trueFalse"
+    }
+
+
+def gen_write_blanks_exercise(text: str, quantity: int, start_id, difficulty):
+    gen_short_answer_questions = "Generate " + str(quantity) + " " + difficulty + " difficulty short answer questions, and the possible answers " \
+                                                               "(max 3 words per answer), about this text: '" + text + "'. " \
+                                                                                                                       "Provide your answer in this JSON format: {\"questions\": [{\"question\": question, " \
+                                                                                                                       "\"possible_answers\": [\"answer_1\", \"answer_2\"]}]}"
+
+    token_count = count_tokens(gen_short_answer_questions)["n_tokens"]
+    questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_short_answer_questions, token_count,
+                                          ["questions"],
+                                          GEN_QUESTION_TEMPERATURE)["questions"][:quantity]
+
+    return {
+        "id": str(uuid.uuid4()),
+        "maxWords": 3,
+        "prompt": "Choose no more than three words and/or a number from the passage for each answer.",
+        "solutions": build_write_blanks_solutions(questions, start_id),
+        "text": build_write_blanks_text(questions, start_id),
+        "type": "writeBlanks"
+    }
+
+
+def gen_multiple_choice_exercise_listening_conversation(text: str, quantity: int, start_id, difficulty):
+    gen_multiple_choice_for_text = "Generate " + str(
+        quantity) + " " + difficulty + " difficulty multiple choice questions of 4 options of for this conversation: " \
+                    "'" + text + "'"
+    token_count = count_tokens(gen_multiple_choice_for_text)["n_tokens"]
+    mc_questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_multiple_choice_for_text, token_count,
+                                             None,
+                                             GEN_QUESTION_TEMPERATURE)
+    parse_mc_questions = "Parse the questions into this json format: {\"questions\": [{\"id\": \"9\", \"options\": [{\"id\": \"A\", \"text\": " \
+                         "\"Economic benefits\"}, {\"id\": \"B\", \"text\": \"Government regulations\"}, {\"id\": \"C\", \"text\": " \
+                         "\"Concerns about climate change\"}, {\"id\": \"D\", \"text\": \"Technological advancement\"}], " \
+                         "\"prompt\": \"What is the main reason for the shift towards renewable energy sources?\", " \
+                         "\"solution\": \"C\", \"variant\": \"text\"}]}. \nThe questions: '" + mc_questions + "'"
+    token_count = count_tokens(parse_mc_questions)["n_tokens"]
+    question = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count,
+                                         ["questions"],
+                                         GEN_QUESTION_TEMPERATURE)
+    return {
+        "id": str(uuid.uuid4()),
+        "prompt": "Select the appropriate option.",
+        "questions": fix_exercise_ids(question, start_id)["questions"],
+        "type": "multipleChoice",
+    }
+
+
+def gen_multiple_choice_exercise_listening_monologue(text: str, quantity: int, start_id, difficulty):
+    gen_multiple_choice_for_text = "Generate " + str(quantity) + " " + difficulty + " difficulty multiple choice questions for this monologue: " \
+                                                                 "'" + text + "'"
+    token_count = count_tokens(gen_multiple_choice_for_text)["n_tokens"]
+    mc_questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_multiple_choice_for_text, token_count,
+                                             None,
+                                             GEN_QUESTION_TEMPERATURE)
+    parse_mc_questions = "Parse the questions into this json format: {\"questions\": [{\"id\": \"9\", \"options\": [{\"id\": \"A\", \"text\": " \
+                         "\"Economic benefits\"}, {\"id\": \"B\", \"text\": \"Government regulations\"}, {\"id\": \"C\", \"text\": " \
+                         "\"Concerns about climate change\"}, {\"id\": \"D\", \"text\": \"Technological advancement\"}], " \
+                         "\"prompt\": \"What is the main reason for the shift towards renewable energy sources?\", " \
+                         "\"solution\": \"C\", \"variant\": \"text\"}]}. \nThe questions: '" + mc_questions + "'"
+    token_count = count_tokens(parse_mc_questions)["n_tokens"]
+    question = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count,
+                                         ["questions"],
+                                         GEN_QUESTION_TEMPERATURE)
+    return {
+        "id": str(uuid.uuid4()),
+        "prompt": "Select the appropriate option.",
+        "questions": fix_exercise_ids(question, start_id)["questions"],
+        "type": "multipleChoice",
+    }
+
+
+def gen_write_blanks_questions_exercise_listening_conversation(text: str, quantity: int, start_id, difficulty):
+    gen_write_blanks_questions = "Generate " + str(quantity) + " " + difficulty + " difficulty short answer questions, and the possible answers " \
+                                                               "(max 3 words per answer), about a monologue and" \
+                                                               "respond in this JSON format: {\"questions\": [{\"question\": question, " \
+                                                               "\"possible_answers\": [\"answer_1\", \"answer_2\"]}]}." \
+                                                               "The monologue is this: '" + text + "'"
+
+    token_count = count_tokens(gen_write_blanks_questions)["n_tokens"]
+    questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_write_blanks_questions, token_count,
+                                          ["questions"],
+                                          GEN_QUESTION_TEMPERATURE)["questions"][:quantity]
+
+    return {
+        "id": str(uuid.uuid4()),
+        "maxWords": 3,
+        "prompt": "You will hear a conversation. Answer the questions below using no more than three words or a number accordingly.",
+        "solutions": build_write_blanks_solutions(questions, start_id),
+        "text": build_write_blanks_text(questions, start_id),
+        "type": "writeBlanks"
+    }
+
+
+def gen_write_blanks_questions_exercise_listening_monologue(text: str, quantity: int, start_id, difficulty):
+    gen_write_blanks_questions = "Generate " + str(quantity) + " " + difficulty + " difficulty short answer questions, and the possible answers " \
+                                                               "(max 3 words per answer), about a monologue and" \
+                                                               "respond in this JSON format: {\"questions\": [{\"question\": question, " \
+                                                               "\"possible_answers\": [\"answer_1\", \"answer_2\"]}]}." \
+                                                               "The monologue is this: '" + text + "'"
+
+    token_count = count_tokens(gen_write_blanks_questions)["n_tokens"]
+    questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_write_blanks_questions, token_count,
+                                          ["questions"],
+                                          GEN_QUESTION_TEMPERATURE)["questions"][:quantity]
+
+    return {
+        "id": str(uuid.uuid4()),
+        "maxWords": 3,
+        "prompt": "You will hear a monologue. Answer the questions below using no more than three words or a number accordingly.",
+        "solutions": build_write_blanks_solutions(questions, start_id),
+        "text": build_write_blanks_text(questions, start_id),
+        "type": "writeBlanks"
+    }
+
+
+def gen_write_blanks_notes_exercise_listening_conversation(text: str, quantity: int, start_id, difficulty):
+    gen_write_blanks_notes = "Generate " + str(quantity) + " " + difficulty + " difficulty notes taken from the conversation and and respond in this " \
+                                                           "JSON format: { \"notes\": [\"note_1\", \"note_2\"] }. The monologue is this: '" + text + "'"
+
+    token_count = count_tokens(gen_write_blanks_notes)["n_tokens"]
+    questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_write_blanks_notes, token_count,
+                                          ["notes"],
+                                          GEN_QUESTION_TEMPERATURE)["notes"][:quantity]
+    formatted_phrases = "\n".join([f"{i + 1}. {phrase}" for i, phrase in enumerate(questions)])
+    gen_words_to_replace = "Select 1 word from each phrase in the list and respond in this " \
+                           "JSON format: { \"words\": [\"word_1\", \"word_2\"] }. The phrases are: " + formatted_phrases
+    words = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_words_to_replace, token_count,
+                                      ["words"],
+                                      GEN_QUESTION_TEMPERATURE)["words"][:quantity]
+    replaced_notes = replace_first_occurrences_with_placeholders_notes(questions, words, start_id)
+    return {
+        "id": str(uuid.uuid4()),
+        "maxWords": 3,
+        "prompt": "Fill the blank space with the word missing from the audio.",
+        "solutions": build_write_blanks_solutions_listening(words, start_id),
+        "text": "\\n".join(replaced_notes),
+        "type": "writeBlanks"
+    }
+
+
+def gen_write_blanks_notes_exercise_listening_monologue(text: str, quantity: int, start_id, difficulty):
+    gen_write_blanks_notes = "Generate " + str(quantity) + " " + difficulty + " difficulty notes taken from the monologue and respond in this " \
+                                                           "JSON format: { \"notes\": [\"note_1\", \"note_2\"] }. The monologue is this: '" + text + "'"
+
+    token_count = count_tokens(gen_write_blanks_notes)["n_tokens"]
+    questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_write_blanks_notes, token_count,
+                                          ["notes"],
+                                          GEN_QUESTION_TEMPERATURE)["notes"][:quantity]
+    formatted_phrases = "\n".join([f"{i + 1}. {phrase}" for i, phrase in enumerate(questions)])
+    gen_words_to_replace = "Select 1 word from each phrase in the list and respond in this " \
+                           "JSON format: { \"words\": [\"word_1\", \"word_2\"] }. The phrases are: " + formatted_phrases
+    words = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_words_to_replace, token_count,
+                                      ["words"],
+                                      GEN_QUESTION_TEMPERATURE)["words"][:quantity]
+    replaced_notes = replace_first_occurrences_with_placeholders_notes(questions, words, start_id)
+    return {
+        "id": str(uuid.uuid4()),
+        "maxWords": 3,
+        "prompt": "Fill the blank space with the word missing from the audio.",
+        "solutions": build_write_blanks_solutions_listening(words, start_id),
+        "text": "\\n".join(replaced_notes),
+        "type": "writeBlanks"
+    }
+
+
+def gen_write_blanks_form_exercise_listening_conversation(text: str, quantity: int, start_id, difficulty):
+    gen_write_blanks_form = "Generate a form with " + str(quantity) + " " + difficulty + " difficulty key-value pairs about the conversation. " \
+                                                                      "The conversation is this: '" + text + "'"
+    token_count = count_tokens(gen_write_blanks_form)["n_tokens"]
+    form = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_write_blanks_form, token_count,
+                                     None,
+                                     GEN_QUESTION_TEMPERATURE)
+    parse_form = "Parse the form to this JSON format: { \"form\": [\"string\", \"string\"] }. The form is this: '" + form + "'"
+    token_count = count_tokens(parse_form)["n_tokens"]
+    parsed_form = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_form, token_count,
+                                            ["form"],
+                                            GEN_QUESTION_TEMPERATURE)["form"][:quantity]
+    replaced_form, words = build_write_blanks_text_form(parsed_form, start_id)
+    return {
+        "id": str(uuid.uuid4()),
+        "maxWords": 3,
+        "prompt": "You will hear a conversation. Fill the form with words/numbers missing.",
+        "solutions": build_write_blanks_solutions_listening(words, start_id),
+        "text": replaced_form,
+        "type": "writeBlanks"
+    }
+
+
+def gen_write_blanks_form_exercise_listening_monologue(text: str, quantity: int, start_id, difficulty):
+    gen_write_blanks_form = "Generate a form with " + str(quantity) + " " + difficulty + " difficulty key-value pairs about the monologue. " \
+                                                                      "The monologue is this: '" + text + "'"
+    token_count = count_tokens(gen_write_blanks_form)["n_tokens"]
+    form = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_write_blanks_form, token_count,
+                                     None,
+                                     GEN_QUESTION_TEMPERATURE)
+    parse_form = "Parse the form to this JSON format: { \"form\": [\"string\", \"string\"] }. The form is this: '" + form + "'"
+    token_count = count_tokens(parse_form)["n_tokens"]
+    parsed_form = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_form, token_count,
+                                            ["form"],
+                                            GEN_QUESTION_TEMPERATURE)["form"][:quantity]
+    replaced_form, words = build_write_blanks_text_form(parsed_form, start_id)
+    return {
+        "id": str(uuid.uuid4()),
+        "maxWords": 3,
+        "prompt": "You will hear a monologue. Fill the form with words/numbers missing.",
+        "solutions": build_write_blanks_solutions_listening(words, start_id),
+        "text": replaced_form,
+        "type": "writeBlanks"
+    }
+
+
+def gen_multiple_choice_level(quantity: int, start_id=1):
+    gen_multiple_choice_for_text = "Generate " + str(
+        quantity) + " multiple choice questions of 4 options for an english level exam, some easy questions, some intermediate " \
+                    "questions and some advanced questions. Ensure that the questions cover a range of topics such as " \
+                    "verb tense, subject-verb agreement, pronoun usage, sentence structure, and punctuation. Make sure " \
+                    "every question only has 1 correct answer."
+    token_count = count_tokens(gen_multiple_choice_for_text)["n_tokens"] - 300
+    mc_questions = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_multiple_choice_for_text, token_count,
+                                             None,
+                                             GEN_QUESTION_TEMPERATURE)
+    split_mc_questions = mc_questions.split('13')
+
+    parse_mc_questions = ('Parse the questions into this json format: {"questions": [{"id": "9", "options": '
+                          '[{"id": "A", "text": '
+                          '"And"}, {"id": "B", "text": "Cat"}, {"id": "C", "text": '
+                          '"Happy"}, {"id": "D", "text": "Jump"}], '
+                          '"prompt": "Which of the following is a conjunction?", '
+                          '"solution": "A", "variant": "text"}]}. '
+                          '\nThe questions: "' + split_mc_questions[0] + '"')
+    token_count = count_tokens(parse_mc_questions, model_name=GPT_3_5_TURBO_INSTRUCT)["n_tokens"]
+    question = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count,
+                                         ["questions"],
+                                         GEN_QUESTION_TEMPERATURE)
+    parse_mc_questions = ('Parse the questions into this json format: {"questions": [{"id": "9", "options": '
+                          '[{"id": "A", "text": '
+                          '"And"}, {"id": "B", "text": "Cat"}, {"id": "C", "text": '
+                          '"Happy"}, {"id": "D", "text": "Jump"}], '
+                          '"prompt": "Which of the following is a conjunction?", '
+                          '"solution": "A", "variant": "text"}]}. '
+                          '\nThe questions: "' + '13' + split_mc_questions[1] + '"')
+    token_count = count_tokens(parse_mc_questions, model_name=GPT_3_5_TURBO_INSTRUCT)["n_tokens"]
+    question_2 = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_questions, token_count,
+                                           ["questions"],
+                                           GEN_QUESTION_TEMPERATURE)
+    question["questions"].extend(question_2["questions"])
+
+    all_exams = get_all("level")
+    seen_keys = set()
+    for i in range(len(question["questions"])):
+        question["questions"][i], seen_keys = replace_exercise_if_exists(all_exams, question["questions"][i], question,
+                                                                         seen_keys)
+    return {
+        "id": str(uuid.uuid4()),
+        "prompt": "Select the appropriate option.",
+        "questions": fix_exercise_ids(question, start_id)["questions"],
+        "type": "multipleChoice",
+    }
+
+
+def replace_exercise_if_exists(all_exams, current_exercise, current_exam, seen_keys):
+    # Extracting relevant fields for comparison
+    key = (current_exercise['prompt'], tuple(sorted(option['text'] for option in current_exercise['options'])))
+    # Check if the key is in the set
+    if key in seen_keys:
+        return replace_exercise_if_exists(all_exams, generate_single_mc_level_question(), current_exam, seen_keys)
+    else:
+        seen_keys.add(key)
+
+    for exam in all_exams:
+        exam_dict = exam.to_dict()
+        if any(
+                exercise["prompt"] == current_exercise["prompt"] and
+                any(exercise["options"][0]["text"] == current_option["text"] for current_option in
+                    current_exercise["options"])
+                for exercise in exam_dict.get("exercises", [])[0]["questions"]
+        ):
+            return replace_exercise_if_exists(all_exams, generate_single_mc_level_question(), current_exam, seen_keys)
+    return current_exercise, seen_keys
+
+
+def generate_single_mc_level_question():
+    gen_multiple_choice_for_text = "Generate 1 multiple choice question of 4 options for an english level exam, it can " \
+                                   "be easy, intermediate or advanced."
+    token_count = count_tokens(gen_multiple_choice_for_text)["n_tokens"] - 300
+    mc_question = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, gen_multiple_choice_for_text, token_count,
+                                            None,
+                                            GEN_QUESTION_TEMPERATURE)
+
+    parse_mc_question = ('Parse the question into this json format: {"id": "9", "options": '
+                         '[{"id": "A", "text": '
+                         '"And"}, {"id": "B", "text": "Cat"}, {"id": "C", "text": '
+                         '"Happy"}, {"id": "D", "text": "Jump"}], '
+                         '"prompt": "Which of the following is a conjunction?", '
+                         '"solution": "A", "variant": "text"}. '
+                         '\nThe questions: "' + mc_question + '"')
+
+    token_count = count_tokens(parse_mc_question, model_name=GPT_3_5_TURBO_INSTRUCT)["n_tokens"]
+    question = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, parse_mc_question, token_count,
+                                         ["options"],
+                                         GEN_QUESTION_TEMPERATURE)
+    return question

helper/file_helper.py

@@ -0,0 +1,17 @@
+import os
+import datetime
+from pathlib import Path
+
+
+def delete_files_older_than_one_day(directory):
+    current_time = datetime.datetime.now()
+
+    for entry in os.scandir(directory):
+        if entry.is_file():
+            file_path = Path(entry)
+            file_name = file_path.name
+            file_modified_time = datetime.datetime.fromtimestamp(file_path.stat().st_mtime)
+            time_difference = current_time - file_modified_time
+            if time_difference.days > 1  and "placeholder" not in file_name:
+                file_path.unlink()
+                print(f"Deleted file: {file_path}")

helper/firebase_helper.py

@@ -0,0 +1,90 @@
+import logging
+
+from google.cloud import storage
+import os
+import uuid
+
+import firebase_admin
+from firebase_admin import credentials, firestore
+
+from dotenv import load_dotenv
+
+import app
+
+
+def download_firebase_file(bucket_name, source_blob_name, destination_file_name):
+    # Downloads a file from Firebase Storage.
+    storage_client = storage.Client()
+    bucket = storage_client.bucket(bucket_name)
+    blob = bucket.blob(source_blob_name)
+    blob.download_to_filename(destination_file_name)
+    logging.info(f"File uploaded to {destination_file_name}")
+    return destination_file_name
+
+def upload_file_firebase(bucket_name, destination_blob_name, source_file_name):
+    # Uploads a file to Firebase Storage.
+    storage_client = storage.Client()
+    bucket = storage_client.bucket(bucket_name)
+    try:
+        blob = bucket.blob(destination_blob_name)
+        blob.upload_from_filename(source_file_name)
+        logging.info(f"File uploaded to {destination_blob_name}")
+        return True
+    except Exception as e:
+        app.app.logger.error("Error uploading file to Google Cloud Storage: " + str(e))
+        return False
+
+def upload_file_firebase_get_url(bucket_name, destination_blob_name, source_file_name):
+    # Uploads a file to Firebase Storage.
+    storage_client = storage.Client()
+    bucket = storage_client.bucket(bucket_name)
+    try:
+        blob = bucket.blob(destination_blob_name)
+        blob.upload_from_filename(source_file_name)
+        logging.info(f"File uploaded to {destination_blob_name}")
+
+        # Make the file public
+        blob.make_public()
+
+        # Get the public URL
+        url = blob.public_url
+        return url
+    except Exception as e:
+        app.app.logger.error("Error uploading file to Google Cloud Storage: " + str(e))
+        return None
+
+def save_to_db(collection: str, item):
+    db = firestore.client()
+    collection_ref = db.collection(collection)
+    (update_time, document_ref) = collection_ref.add(item)
+    if document_ref:
+        logging.info(f"Document added with ID: {document_ref.id}")
+        return (True, document_ref.id)
+    else:
+        return (False, None)
+
+
+def save_to_db_with_id(collection: str, item, id: str):
+    db = firestore.client()
+    collection_ref = db.collection(collection)
+    # Reference to the specific document with the desired ID
+    document_ref = collection_ref.document(id)
+    # Set the data to the document
+    document_ref.set(item)
+    if document_ref:
+        logging.info(f"Document added with ID: {document_ref.id}")
+        return (True, document_ref.id)
+    else:
+        return (False, None)
+
+def get_all(collection: str):
+    db = firestore.client()
+    collection_ref = db.collection(collection)
+
+    all_exercises = (
+        collection_ref
+        .get()
+    )
+
+    return all_exercises
+

helper/heygen_api.py

@@ -0,0 +1,180 @@
+import logging
+import os
+import random
+
+import requests
+import time
+
+from dotenv import load_dotenv
+
+import app
+from helper.constants import *
+from helper.firebase_helper import upload_file_firebase_get_url, save_to_db_with_id
+from heygen.AvatarEnum import AvatarEnum
+
+load_dotenv()
+
+# Get HeyGen token
+TOKEN = os.getenv("HEY_GEN_TOKEN")
+
+# POST TO CREATE VIDEO
+CREATE_VIDEO_URL = 'https://api.heygen.com/v1/template.generate'
+GET_VIDEO_URL = 'https://api.heygen.com/v1/video_status.get'
+POST_HEADER = {
+    'X-Api-Key': TOKEN,
+    'Content-Type': 'application/json'
+}
+GET_HEADER = {
+    'X-Api-Key': TOKEN
+}
+
+MATTHEW_NOAH = "11b234e504e44bfda9bc6b7aac3c8f81"
+VERA_CERISE = "9bf2f27009cd403ab4ba4e22629b27bb"
+EDWARD_TONY = "d3333e37952946059b45efc8482a2b6c"
+TANYA_MOLLY = "07c75076b3f94df4ac658c6de72be83a"
+KAYLA_ABBI = "d688099f8db9472cb4890b0561e81793"
+JEROME_RYAN = "ad41feb2a5c4483085525e3d8907f512"
+TYLER_CHRISTOPHER = "03c796f8ed274bb38f19e893bcbc6121"
+
+
+def create_videos_and_save_to_db(exercises, template, id):
+    # Speaking 1
+    # Using list comprehension to find the element with the desired value in the 'type' field
+    found_exercises_1 = [element for element in exercises if element.get('type') == 1]
+    # Check if any elements were found
+    if found_exercises_1:
+        exercise_1 = found_exercises_1[0]
+        app.app.logger.info('Creating video for speaking part 1')
+        sp1_result = create_video(exercise_1["question"], random.choice(list(AvatarEnum)))
+        if sp1_result is not None:
+            sound_file_path = VIDEO_FILES_PATH + sp1_result
+            firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp1_result
+            url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+            sp1_video_path = firebase_file_path
+            sp1_video_url = url
+            template["exercises"][0]["text"] = exercise_1["question"]
+            template["exercises"][0]["title"] = exercise_1["topic"]
+            template["exercises"][0]["video_url"] = sp1_video_url
+            template["exercises"][0]["video_path"] = sp1_video_path
+        else:
+            app.app.logger.error("Failed to create video for part 1 question: " + exercise_1["question"])
+
+    # Speaking 2
+    # Using list comprehension to find the element with the desired value in the 'type' field
+    found_exercises_2 = [element for element in exercises if element.get('type') == 2]
+    # Check if any elements were found
+    if found_exercises_2:
+        exercise_2 = found_exercises_2[0]
+        app.app.logger.info('Creating video for speaking part 2')
+        sp2_result = create_video(exercise_2["question"], random.choice(list(AvatarEnum)))
+        if sp2_result is not None:
+            sound_file_path = VIDEO_FILES_PATH + sp2_result
+            firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp2_result
+            url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+            sp2_video_path = firebase_file_path
+            sp2_video_url = url
+            template["exercises"][1]["prompts"] = exercise_2["prompts"]
+            template["exercises"][1]["text"] = exercise_2["question"]
+            template["exercises"][1]["title"] = exercise_2["topic"]
+            template["exercises"][1]["video_url"] = sp2_video_url
+            template["exercises"][1]["video_path"] = sp2_video_path
+        else:
+            app.app.logger.error("Failed to create video for part 2 question: " + exercise_2["question"])
+
+    # Speaking 3
+    # Using list comprehension to find the element with the desired value in the 'type' field
+    found_exercises_3 = [element for element in exercises if element.get('type') == 3]
+    # Check if any elements were found
+    if found_exercises_3:
+        exercise_3 = found_exercises_3[0]
+        sp3_questions = []
+        avatar = random.choice(list(AvatarEnum))
+        app.app.logger.info('Creating videos for speaking part 3')
+        for question in exercise_3["questions"]:
+            result = create_video(question, avatar)
+            if result is not None:
+                sound_file_path = VIDEO_FILES_PATH + result
+                firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + result
+                url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+                video = {
+                    "text": question,
+                    "video_path": firebase_file_path,
+                    "video_url": url
+                }
+                sp3_questions.append(video)
+            else:
+                app.app.logger.error("Failed to create video for part 3 question: " + question)
+        template["exercises"][2]["prompts"] = sp3_questions
+        template["exercises"][2]["title"] = exercise_3["topic"]
+
+    if not found_exercises_3:
+        template["exercises"].pop(2)
+    if not found_exercises_2:
+        template["exercises"].pop(1)
+    if not found_exercises_1:
+        template["exercises"].pop(0)
+
+    save_to_db_with_id("speaking", template, id)
+    app.app.logger.info('Saved speaking to DB with id ' + id + " : " + str(template))
+
+def create_video(text, avatar):
+    # POST TO CREATE VIDEO
+    create_video_url = 'https://api.heygen.com/v2/template/' + avatar + '/generate'
+    data = {
+        "test": False,
+        "caption": False,
+        "title": "video_title",
+        "variables": {
+            "script_here": {
+                "name": "script_here",
+                "type": "text",
+                "properties": {
+                    "content": text
+                }
+            }
+        }
+    }
+    response = requests.post(create_video_url, headers=POST_HEADER, json=data)
+    app.app.logger.info(response.status_code)
+    app.app.logger.info(response.json())
+
+    # GET TO CHECK STATUS AND GET VIDEO WHEN READY
+    video_id = response.json()["data"]["video_id"]
+    params = {
+        'video_id': response.json()["data"]["video_id"]
+    }
+    response = {}
+    status = "processing"
+    error = None
+
+    while status != "completed" and error is None:
+        response = requests.get(GET_VIDEO_URL, headers=GET_HEADER, params=params)
+        response_data = response.json()
+
+        status = response_data["data"]["status"]
+        error = response_data["data"]["error"]
+
+        if status != "completed" and error is None:
+            app.app.logger.info(f"Status: {status}")
+            time.sleep(10)  # Wait for 10 second before the next request
+
+    app.app.logger.info(response.status_code)
+    app.app.logger.info(response.json())
+
+    # DOWNLOAD VIDEO
+    download_url = response.json()['data']['video_url']
+    output_directory = 'download-video/'
+    output_filename = video_id + '.mp4'
+
+    response = requests.get(download_url)
+
+    if response.status_code == 200:
+        os.makedirs(output_directory, exist_ok=True)  # Create the directory if it doesn't exist
+        output_path = os.path.join(output_directory, output_filename)
+        with open(output_path, 'wb') as f:
+            f.write(response.content)
+        app.app.logger.info(f"File '{output_filename}' downloaded successfully.")
+        return output_filename
+    else:
+        app.app.logger.error(f"Failed to download file. Status code: {response.status_code}")
+        return None

helper/openai_interface.py

@@ -1,48 +1,155 @@
 import json
 import openai
 import os
+import re
 
 from dotenv import load_dotenv
 
+from helper.constants import GPT_3_5_TURBO_INSTRUCT, BLACKLISTED_WORDS
+from helper.token_counter import count_tokens
+
 load_dotenv()
 openai.api_key = os.getenv("OPENAI_API_KEY")
 
-
 MAX_TOKENS = 4097
 TOP_P = 0.9
 FREQUENCY_PENALTY = 0.5
 
-TRY_LIMIT = 1
+TRY_LIMIT = 2
 
 try_count = 0
-def process_response(input_string):
-    json_obj = {}
-    parsed_string = input_string.replace("'", "\"")
-    parsed_string = parsed_string.replace("\n\n", " ")
-    try:
-        json_obj = json.loads(parsed_string)
-    except json.JSONDecodeError:
-        print("Invalid JSON string!")
 
-    return json_obj
+# GRADING SUMMARY
+chat_config = {'max_tokens': 1000, 'temperature': 0.2}
+section_keys = ['reading', 'listening', 'writing', 'speaking', 'level']
+grade_top_limit = 9
+
+tools = [{
+    "type": "function",
+    "function": {
+        "name": "save_evaluation_and_suggestions",
+        "description": "Saves the evaluation and suggestions requested by input.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "evaluation": {
+                    "type": "string",
+                    "description": "A comment on the IELTS section grade obtained in the specific section and what it could mean without suggestions.",
+                },
+                "suggestions": {
+                    "type": "string",
+                    "description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.",
+                },
+            },
+            "required": ["evaluation", "suggestions"],
+        },
+    }
+}]
+
+
+###
+
+def process_response(input_string, quotation_check_field):
+    if '{' in input_string:
+        try:
+            # Find the index of the first occurrence of '{'
+            index = input_string.index('{')
+            # Extract everything after the first '{' (inclusive)
+            result = input_string[index:]
+            if re.search(r"'" + quotation_check_field + "':\s*'(.*?)'", result, re.DOTALL | re.MULTILINE) or \
+                    re.search(r"'" + quotation_check_field + "':\s*\[([^\]]+)]", result, re.DOTALL | re.MULTILINE):
+                json_obj = json.loads(parse_string(result))
+                return json_obj
+            else:
+                parsed_string = result.replace("\n\n", " ")
+                parsed_string = parsed_string.replace("\n", " ")
+                parsed_string = re.sub(r',\s*]', ']', parsed_string)
+                parsed_string = re.sub(r',\s*}', '}', parsed_string)
+                if (parsed_string.find('[') == -1) and (parsed_string.find(']') == -1):
+                    parsed_string = parse_string_2(parsed_string)
+                    return json.loads(parsed_string)
+
+                return json.loads(parsed_string)
+        except Exception as e:
+            print(f"Invalid JSON string! Exception: {e}")
+            print(f"String: {input_string}")
+            print(f"Exception: {e}")
+    else:
+        return input_string
+
+
+def parse_string(to_parse: str):
+    parsed_string = to_parse.replace("\"", "\\\"")
+    pattern = r"(?<!\w)'|'(?!\w)"
+    parsed_string = re.sub(pattern, '"', parsed_string)
+    parsed_string = parsed_string.replace("\\\"", "'")
+    parsed_string = parsed_string.replace("\n\n", " ")
+    parsed_string = re.sub(r',\s*]', ']', parsed_string)
+    parsed_string = re.sub(r',\s*}', '}', parsed_string)
+    return parsed_string
+
+
+def parse_string_2(to_parse: str):
+    keys_and_values_str = to_parse.replace("{", "").replace("}", "")
+    split_pattern = r'(?<="),|(?<="):'
+    keys_and_values = re.split(split_pattern, keys_and_values_str)
+
+    keys = []
+    values = []
+
+    for idx, x in enumerate(keys_and_values):
+        if (idx % 2) == 0:
+            keys.append(x)
+        else:
+            values.append(x)
+
+    parsed_values = []
+
+    for value in values:
+        parsed_values.append(("\"" + value.replace("\"", "").strip() + "\""))
+
+    for ind, parsed_value in enumerate(parsed_values):
+        to_parse = to_parse.replace(values[ind], parsed_values[ind])
+
+    to_parse = to_parse.replace(":", ": ")
+    return to_parse
+
+
+def remove_special_chars_and_escapes(input_string):
+    parsed_string = input_string.replace("\\\"", "'")
+    parsed_string = parsed_string.replace("\n\n", " ")
+    # Define a regular expression pattern to match special characters and escapes
+    pattern = r'(\\[nrt])|[^a-zA-Z0-9\s]'
+
+    # Use re.sub() to replace the matched patterns with an empty string
+    cleaned_string = re.sub(pattern, '', parsed_string)
+
+    return cleaned_string
+
 
 def check_fields(obj, fields):
     return all(field in obj for field in fields)
 
-def make_openai_call(messages, token_count, fields_to_check, temperature):
+
+def make_openai_call(model, messages, token_count, fields_to_check, temperature):
     global try_count
     result = openai.ChatCompletion.create(
-        model="gpt-3.5-turbo",
+        model=model,
         max_tokens=int(MAX_TOKENS - token_count - 300),
         temperature=float(temperature),
         top_p=float(TOP_P),
         frequency_penalty=float(FREQUENCY_PENALTY),
         messages=messages
     )
-    processed_response = process_response(result["choices"][0]["message"]["content"])
+
+    if fields_to_check is None:
+        return result["choices"][0]["message"]["content"]
+
+    processed_response = process_response(result["choices"][0]["message"]["content"], fields_to_check[0])
+
     if check_fields(processed_response, fields_to_check) is False and try_count < TRY_LIMIT:
         try_count = try_count + 1
-        return make_openai_call(messages, token_count, fields_to_check)
+        return make_openai_call(model, messages, token_count, fields_to_check, temperature)
     elif try_count >= TRY_LIMIT:
         try_count = 0
         return result["choices"][0]["message"]["content"]
@@ -51,3 +158,120 @@ def make_openai_call(messages, token_count, fields_to_check, temperature):
         return processed_response
 
 
+def make_openai_instruct_call(model, message: str, token_count, fields_to_check, temperature):
+    global try_count
+    response = openai.Completion.create(
+        model=model,
+        prompt=message,
+        max_tokens=int(4097 - token_count - 300),
+        temperature=0.7
+    )["choices"][0]["text"]
+
+    if has_blacklisted_words(response) and try_count < TRY_LIMIT:
+        try_count = try_count + 1
+        return make_openai_instruct_call(model, message, token_count, fields_to_check, temperature)
+    elif has_blacklisted_words(response) and try_count >= TRY_LIMIT:
+        return ""
+
+    if fields_to_check is None:
+        return response.replace("\n\n", " ").strip()
+
+    processed_response = process_response(response, fields_to_check[0])
+    if check_fields(processed_response, fields_to_check) is False and try_count < TRY_LIMIT:
+        try_count = try_count + 1
+        return make_openai_instruct_call(model, message, token_count, fields_to_check, temperature)
+    else:
+        try_count = 0
+        return processed_response
+
+
+# GRADING SUMMARY
+def calculate_grading_summary(body):
+    extracted_sections = extract_existing_sections_from_body(body, section_keys)
+
+    ret = []
+
+    for section in extracted_sections:
+        openai_response_dict = calculate_section_grade_summary(section)
+        ret = ret + [{'code': section['code'], 'name': section['name'], 'grade': section['grade'],
+                      'evaluation': openai_response_dict['evaluation'],
+                      'suggestions': openai_response_dict['suggestions']}]
+
+    return {'sections': ret}
+
+
+def calculate_section_grade_summary(section):
+    messages = [
+        {
+            "role": "user",
+            "content": "You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the grade obtained in the section. You should offer a evaluation comment on this grade and separately suggestions on how to possibly get a better grade.",
+        },
+        {
+            "role": "user",
+            "content": "Section: " + str(section['name']) + " Grade: " + str(section['grade']),
+        },
+        {"role": "user", "content": "Speak in third person."},
+        {"role": "user",
+         "content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section."},
+        {"role": "user",
+         "content": "Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, be one paragraph long. "},
+        {"role": "user", "content": "Please save the evaluation comment and suggestions generated."}
+    ]
+
+    if section['code'] == "level":
+        messages[2:2] = [{
+            "role": "user",
+            "content": "This section is comprised of multiple choice questions that measure the user's overall english level. These multiple choice questions are about knowledge on vocabulary, syntax, grammar rules, and contextual usage. The grade obtained measures the ability in these areas and english language overall."
+        }]
+    elif section['code'] == "speaking":
+        messages[2:2] = [{"role": "user",
+                          "content": "This section is s designed to assess the English language proficiency of individuals who want to study or work in English-speaking countries. The speaking section evaluates a candidate's ability to communicate effectively in spoken English."}]
+
+    res = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        max_tokens=chat_config['max_tokens'],
+        temperature=chat_config['temperature'],
+        tools=tools,
+        messages=messages)
+
+    return parse_openai_response(res)
+
+
+def parse_openai_response(response):
+    if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][
+        0] and 'tool_calls' in response['choices'][0]['message'] and isinstance(
+        response['choices'][0]['message']['tool_calls'], list) and len(
+        response['choices'][0]['message']['tool_calls']) > 0 and \
+            response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
+        return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
+    else:
+        return {'evaluation': "", 'suggestions': ""}
+
+
+def extract_existing_sections_from_body(my_dict, keys_to_extract):
+    if 'sections' in my_dict and isinstance(my_dict['sections'], list) and len(my_dict['sections']) > 0:
+        return list(filter(
+            lambda item: 'code' in item and item['code'] in keys_to_extract and 'grade' in item and 'name' in item,
+            my_dict['sections']))
+
+
+def get_fixed_text(text):
+    message = ('Fix the errors in the given text and put it in a JSON. Do not complete the answer, only replace what '
+               'is wrong. Sample JSON: {"fixed_text": "fixed test with no '
+               'misspelling errors"}] \n The text: "' + text + '"')
+    token_count = count_tokens(message)["n_tokens"]
+    response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, ["fixed_text"], 0.2)
+    return response["fixed_text"]
+
+def get_speaking_corrections(text):
+    message = ('Fix the errors in the provided transcription and put it in a JSON. Do not complete the answer, only '
+               'replace what is wrong. Sample JSON: {"fixed_text": "fixed '
+               'transcription with no misspelling errors"}] \n The text: "' + text + '"')
+    token_count = count_tokens(message)["n_tokens"]
+    response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, ["fixed_text"], 0.2)
+    return response["fixed_text"]
+
+
+def has_blacklisted_words(text: str):
+    text_lower = text.lower()
+    return any(word in text_lower for word in BLACKLISTED_WORDS)

helper/speech_to_text_helper.py

@@ -0,0 +1,104 @@
+import whisper
+import os
+import nltk
+import boto3
+import random
+
+nltk.download('words')
+from nltk.corpus import words
+from helper.constants import *
+
+def speech_to_text(file_path):
+    if os.path.exists(file_path):
+        model = whisper.load_model("base")
+        result = model.transcribe(file_path, fp16=False, language='English', verbose=False)
+        return result["text"]
+    else:
+        print("File not found:", file_path)
+        raise Exception("File " + file_path + " not found.")
+
+def text_to_speech(text: str, file_name: str):
+    # Initialize the Amazon Polly client
+    client = boto3.client(
+        'polly',
+        region_name='eu-west-1',
+        aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
+        aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY")
+    )
+    voice = random.choice(ALL_NEURAL_VOICES)['Id']
+    # Initialize an empty list to store audio segments
+    audio_segments = []
+    for part in divide_text(text):
+        tts_response = client.synthesize_speech(
+            Engine="neural",
+            Text=part,
+            OutputFormat="mp3",
+            VoiceId=voice
+        )
+        audio_segments.append(tts_response['AudioStream'].read())
+    # Combine the audio segments into a single audio file
+    combined_audio = b"".join(audio_segments)
+    # Save the combined audio to a single file
+    with open(file_name, "wb") as f:
+        f.write(combined_audio)
+
+    print("Speech segments saved to " + file_name)
+
+def conversation_text_to_speech(conversation: list, file_name: str):
+    # Initialize the Amazon Polly client
+    client = boto3.client(
+        'polly',
+        region_name='eu-west-1',
+        aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
+        aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY")
+    )
+    # Initialize an empty list to store audio segments
+    audio_segments = []
+    # Iterate through the text segments, convert to audio segments, and store them
+    for segment in conversation:
+        response = client.synthesize_speech(
+            Engine="neural", 
+            Text=segment["text"],
+            OutputFormat="mp3",
+            VoiceId=segment["voice"]
+        )
+        audio_segments.append(response['AudioStream'].read())
+    # Combine the audio segments into a single audio file
+    combined_audio = b"".join(audio_segments)
+    # Save the combined audio to a single file
+    with open(file_name, "wb") as f:
+        f.write(combined_audio)
+
+    print("Speech segments saved to " + file_name)
+
+def has_words(text: str):
+    english_words = set(words.words())
+    words_in_input = text.split()
+    return any(word.lower() in english_words for word in words_in_input)
+
+def has_10_words(text: str):
+    english_words = set(words.words())
+    words_in_input = text.split()
+    english_word_count = sum(1 for word in words_in_input if word.lower() in english_words)
+    return english_word_count >= 10
+
+def divide_text(text, max_length=3000):
+    if len(text) <= max_length:
+        return [text]
+
+    divisions = []
+    current_position = 0
+
+    while current_position < len(text):
+        next_position = min(current_position + max_length, len(text))
+        next_period_position = text.rfind('.', current_position, next_position)
+
+        if next_period_position != -1 and next_period_position > current_position:
+            divisions.append(text[current_position:next_period_position + 1])
+            current_position = next_period_position + 1
+        else:
+            # If no '.' found in the next chunk, split at max_length
+            divisions.append(text[current_position:next_position])
+            current_position = next_position
+
+    return divisions

heygen/AvatarEnum.py

@@ -0,0 +1,11 @@
+from enum import Enum
+
+
+class AvatarEnum(Enum):
+    MATTHEW_NOAH = "5912afa7c77c47d3883af3d874047aaf"
+    VERA_CERISE = "9e58d96a383e4568a7f1e49df549e0e4"
+    EDWARD_TONY = "d2cdd9c0379a4d06ae2afb6e5039bd0c"
+    TANYA_MOLLY = "045cb5dcd00042b3a1e4f3bc1c12176b"
+    KAYLA_ABBI = "1ae1e5396cc444bfad332155fdb7a934"
+    JEROME_RYAN = "0ee6aa7cc1084063a630ae514fccaa31"
+    TYLER_CHRISTOPHER = "5772cff935844516ad7eeff21f839e43"

heygen/filter_json.py

@@ -0,0 +1,18 @@
+import json
+
+# Read JSON from a file
+input_filename = "english_voices.json"
+output_filename = "free_english_voices.json"
+
+with open(input_filename, "r") as json_file:
+    data = json.load(json_file)
+
+# Filter entries based on "language": "English"
+filtered_list = [entry for entry in data["data"]["list"] if not entry["is_paid"]]
+data["data"]["list"] = filtered_list
+
+# Write filtered JSON to a new file
+with open(output_filename, "w") as json_file:
+    json.dump(data, json_file, indent=2)
+
+print(f"Filtered JSON written to '{output_filename}'.")

run.py

@@ -1,5 +0,0 @@
-from streamlit.web import bootstrap
-
-real_s_script = 'sp1_playground.py'
-real_w_script = 'wt2_playground.py'
-bootstrap.run(real_s_script, f'run.py {real_s_script}', [], {})

sp1_playground.py

@@ -1,109 +0,0 @@
-import openai
-import os
-from dotenv import load_dotenv
-import whisper
-
-load_dotenv()
-openai.api_key = os.getenv("OPENAI_API_KEY")
-
-def correct_answer(
-        max_tokens,
-        temperature,
-        top_p,
-        frequency_penalty,
-        question_type,
-        question,
-        answer_path
-):
-    model = whisper.load_model("base")
-    # result = model.transcribe("audio-samples/mynameisjeff.wav", fp16=False, language='English', verbose=True)
-    if os.path.exists(answer_path):
-        result = model.transcribe(answer_path, fp16=False, language='English', verbose=True)
-        answer = result["text"]
-        print(answer)
-        res = openai.ChatCompletion.create(
-            model="gpt-3.5-turbo",
-            max_tokens=int(max_tokens),
-            temperature=float(temperature),
-            top_p=float(top_p),
-            frequency_penalty=float(frequency_penalty),
-            messages=
-            [
-                {
-                    "role": "system",
-                    "content": "You are a IELTS examiner.",
-                },
-                {
-                    "role": "system",
-                    "content": f"The question you have to grade is of type {question_type} and is the following: {question}",
-                },
-                {
-                    "role": "system",
-                    "content": "Please provide a JSON object response with the overall grade and breakdown grades, "
-                               "formatted as follows: {'overall': 7.0, 'task_response': {'Fluency and Coherence': 8.0, "
-                               "'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': "
-                               "6.0}}",
-                },
-                {
-                    "role": "system",
-                    "content": "Don't give explanations for the grades, just provide the json with the grades.",
-                },
-                {
-                    "role": "system",
-                    "content": "If the answer is unrelated to the question give it the minimum grade.",
-                },
-                {
-                    "role": "user",
-                    "content": f"Evaluate this answer according to ielts grading system: {answer}",
-                },
-            ],
-        )
-        return res["choices"][0]["message"]["content"]
-    else:
-        print("File not found:", answer_path)
-
-import streamlit as st
-
-# Set the application title
-st.title("GPT-3.5 IELTS Examiner")
-
-# Selection box to select the question type
-question_type = st.selectbox(
-    "What is the question type?",
-    (
-        "Speaking Part 1",
-        "Speaking Part 2",
-        "Speaking Part 3"
-    ),
-)
-
-# Provide the input area for question to be answered
-# PT-1: How do you usually spend your weekends? Why?
-# PT-2: Describe someone you know who does something well. You should say who this person is, how do you know this person, what they do well and explain why you think this person is so good at doing this.
-question = st.text_area("Enter the question:", height=100)
-
-# Provide the input area for text to be summarized
-# audio-samples/mynameisjeff.wav
-answer_path = st.text_area("Enter the answer path:", height=100)
-
-# Initiate two columns for section to be side-by-side
-# col1, col2 = st.columns(2)
-
-# Slider to control the model hyperparameter
-# with col1:
-token = st.slider("Token", min_value=0.0, max_value=2000.0, value=1000.0, step=1.0)
-temp = st.slider("Temperature", min_value=0.0, max_value=1.0, value=0.7, step=0.01)
-top_p = st.slider("Top_p", min_value=0.0, max_value=1.0, value=0.9, step=0.01)
-f_pen = st.slider("Frequency Penalty", min_value=-1.0, max_value=1.0, value=0.5, step=0.01)
-
-# Showing the current parameter used for the model
-# with col2:
-with st.expander("Current Parameter"):
-    st.write("Current Token :", token)
-    st.write("Current Temperature :", temp)
-    st.write("Current Nucleus Sampling :", top_p)
-    st.write("Current Frequency Penalty :", f_pen)
-
-# Creating button for execute the text summarization
-if st.button("Grade"):
-    st.write(correct_answer(token, temp, top_p, f_pen, question_type, question, answer_path))

sp2_playground.py

@@ -1,102 +0,0 @@
-import openai
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-openai.api_key = os.getenv("OPENAI_API_KEY")
-
-
-def generate_summarizer(
-        max_tokens,
-        temperature,
-        top_p,
-        frequency_penalty,
-        question_type,
-        question,
-        answer
-):
-    res = openai.ChatCompletion.create(
-        model="gpt-3.5-turbo",
-        max_tokens=int(max_tokens),
-        temperature=float(temperature),
-        top_p=float(top_p),
-        frequency_penalty=float(frequency_penalty),
-        messages=
-        [
-            {
-                "role": "system",
-                "content": "You are a IELTS examiner.",
-            },
-            {
-                "role": "system",
-                "content": f"The question you have to grade is of type {question_type} and is the following: {question}",
-            },
-            {
-                "role": "system",
-                "content": "Please provide a JSON object response with the overall grade and breakdown grades, "
-                           "formatted as follows: {'overall': 7.0, 'task_response': {'Task Achievement': 8.0, "
-                           "'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, 'Grammatical Range and Accuracy': "
-                           "6.0}}",
-            },
-            {
-                "role": "system",
-                "content": "Don't give explanations for the grades, just provide the json with the grades.",
-            },
-            {
-                "role": "user",
-                "content": f"Evaluate this answer according to ielts grading system: {answer}",
-            },
-        ],
-    )
-    return res["choices"][0]["message"]["content"]
-
-
-import streamlit as st
-
-# Set the application title
-st.title("GPT-3.5 IELTS Examiner")
-
-# qt_col, q_col = st.columns(2)
-
-# Selection box to select the question type
-# with qt_col:
-question_type = st.selectbox(
-    "What is the question type?",
-    (
-        "Listening",
-        "Reading",
-        "Writing Task 1",
-        "Writing Task 2",
-        "Speaking Part 1",
-        "Speaking Part 2"
-    ),
-)
-
-# Provide the input area for question to be answered
-# with q_col:
-question = st.text_area("Enter the question:", height=100)
-
-# Provide the input area for text to be summarized
-answer = st.text_area("Enter the answer:", height=100)
-
-# Initiate two columns for section to be side-by-side
-# col1, col2 = st.columns(2)
-
-# Slider to control the model hyperparameter
-# with col1:
-token = st.slider("Token", min_value=0.0, max_value=2000.0, value=1000.0, step=1.0)
-temp = st.slider("Temperature", min_value=0.0, max_value=1.0, value=0.7, step=0.01)
-top_p = st.slider("Top_p", min_value=0.0, max_value=1.0, value=0.9, step=0.01)
-f_pen = st.slider("Frequency Penalty", min_value=-1.0, max_value=1.0, value=0.5, step=0.01)
-
-# Showing the current parameter used for the model
-# with col2:
-with st.expander("Current Parameter"):
-    st.write("Current Token :", token)
-    st.write("Current Temperature :", temp)
-    st.write("Current Nucleus Sampling :", top_p)
-    st.write("Current Frequency Penalty :", f_pen)
-
-# Creating button for execute the text summarization
-if st.button("Grade"):
-    st.write(generate_summarizer(token, temp, top_p, f_pen, question_type, question, answer))

testing.py

@@ -1,37 +0,0 @@
-from functools import reduce
-from helper.token_counter import count_tokens
-
-# model = whisper.load_model("base")
-# file_path = "audio-samples/mynameisjeff.wav"
-# audio_file = AudioSegment.from_file(file_path)
-# if os.path.exists(file_path):
-#     result = model.transcribe(file_path, fp16=False, language='English', verbose=True)
-#     print(result["text"])
-# else:
-#     print("File not found:", file_path)
-
-messages = [
-    {
-        "role": "system",
-        "content": "You are a IELTS examiner.",
-    },
-    {
-        "role": "system",
-        "content": f"The question you have to grade is of type and is the following: ",
-    },
-    {
-        "role": "system",
-        "content": "Please provide a JSON object response with the overall grade and breakdown grades, "
-                   "formatted as follows: {'overall': 7.0, 'task_response': {'Task Achievement': 8.0, "
-                   "'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, 'Grammatical Range and Accuracy': "
-                   "6.0}}",
-    },
-    {
-        "role": "user",
-        "content": f"Evaluate this answer according to ielts grading system:",
-    },
-]
-token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'],
-                     map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0)
-
-print(token_count)

text-samples/listening.txt

@@ -0,0 +1 @@
+"Provide me with an transcript similar to the ones in ielts exam Listening Section 1. After the transcript, please generate a 'form like' fill in the blanks exercise with 6 items (ex: name, date of birth) to fill related to the text content. Finally provide the answers for the exercise. The response must be a json following this format: { 'type': '<type of registration (ex: hotel, gym, english course, etc)>', 'transcript': '<transcript of just the conversation about a registration of some sort, identify the person talking in each speech line>', 'exercise': { 'item': { '1': '<item 1>', '2': '<item 2>', '3': '<item 3>', '4': '<item 4>', '5': '<item 5>', '6': '<item 5>' }, 'answers': {'1': '<answer to fill blank space in item 1>', '2': '<answer to fill blank space in item 2>', '3': '<answer to fill blank space in item 3>', '4': '<answer to fill blank space in item 4>', '5': '<answer to fill blank space in item 5>', '6': '<answer to fill blank space in item 6>'}}}

text-samples/writing-pt2.txt

@@ -1,6 +1,4 @@
-Q: It is important for children to learn the difference between right and wrong at an early age. Punishment is necessary to help them learn this distinction.
-To what extent do you agree or disagree with this opinion?
-What sort of punishment should parents and teachers be allowed to use to teach good behaviour to children?
+Q: It is important for children to learn the difference between right and wrong at an early age. Punishment is necessary to help them learn this distinction. To what extent do you agree or disagree with this opinion? What sort of punishment should parents and teachers be allowed to use to teach good behaviour to children?
 
 
 A: In today's world, moral values and ethics play a vital role in shaping the character of an individual. Children are the building blocks of society, and it is important to inculcate a sense of right and wrong in them from an early age. While punishment can be an effective tool in teaching the difference between right and wrong, it should not be the only approach. In my opinion, punishment should be used in moderation, and parents and teachers should focus on positive reinforcement and guidance to teach good behavior.

wt1_playground.py

@@ -1,102 +0,0 @@
-import openai
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-openai.api_key = os.getenv("OPENAI_API_KEY")
-
-
-def generate_summarizer(
-        max_tokens,
-        temperature,
-        top_p,
-        frequency_penalty,
-        question_type,
-        question,
-        answer
-):
-    res = openai.ChatCompletion.create(
-        model="gpt-3.5-turbo",
-        max_tokens=int(max_tokens),
-        temperature=float(temperature),
-        top_p=float(top_p),
-        frequency_penalty=float(frequency_penalty),
-        messages=
-        [
-            {
-                "role": "system",
-                "content": "You are a IELTS examiner.",
-            },
-            {
-                "role": "system",
-                "content": f"The question you have to grade is of type {question_type} and is the following: {question}",
-            },
-            {
-                "role": "system",
-                "content": "Please provide a JSON object response with the overall grade and breakdown grades, "
-                           "formatted as follows: {'overall': 7.0, 'task_response': {'Task Achievement': 8.0, "
-                           "'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, 'Grammatical Range and Accuracy': "
-                           "6.0}}",
-            },
-            {
-                "role": "system",
-                "content": "Don't give explanations for the grades, just provide the json with the grades.",
-            },
-            {
-                "role": "user",
-                "content": f"Evaluate this answer according to ielts grading system: {answer}",
-            },
-        ],
-    )
-    return res["choices"][0]["message"]["content"]
-
-
-import streamlit as st
-
-# Set the application title
-st.title("GPT-3.5 IELTS Examiner")
-
-# qt_col, q_col = st.columns(2)
-
-# Selection box to select the question type
-# with qt_col:
-question_type = st.selectbox(
-    "What is the question type?",
-    (
-        "Listening",
-        "Reading",
-        "Writing Task 1",
-        "Writing Task 2",
-        "Speaking Part 1",
-        "Speaking Part 2"
-    ),
-)
-
-# Provide the input area for question to be answered
-# with q_col:
-question = st.text_area("Enter the question:", height=100)
-
-# Provide the input area for text to be summarized
-answer = st.text_area("Enter the answer:", height=100)
-
-# Initiate two columns for section to be side-by-side
-# col1, col2 = st.columns(2)
-
-# Slider to control the model hyperparameter
-# with col1:
-token = st.slider("Token", min_value=0.0, max_value=2000.0, value=1000.0, step=1.0)
-temp = st.slider("Temperature", min_value=0.0, max_value=1.0, value=0.7, step=0.01)
-top_p = st.slider("Top_p", min_value=0.0, max_value=1.0, value=0.9, step=0.01)
-f_pen = st.slider("Frequency Penalty", min_value=-1.0, max_value=1.0, value=0.5, step=0.01)
-
-# Showing the current parameter used for the model
-# with col2:
-with st.expander("Current Parameter"):
-    st.write("Current Token :", token)
-    st.write("Current Temperature :", temp)
-    st.write("Current Nucleus Sampling :", top_p)
-    st.write("Current Frequency Penalty :", f_pen)
-
-# Creating button for execute the text summarization
-if st.button("Grade"):
-    st.write(generate_summarizer(token, temp, top_p, f_pen, question_type, question, answer))

wt2_playground.py

@@ -1,99 +0,0 @@
-import streamlit as st
-import openai
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-openai.api_key = os.getenv("OPENAI_API_KEY")
-
-
-def generate_summarizer(
-        max_tokens,
-        temperature,
-        top_p,
-        frequency_penalty,
-        question_type,
-        question,
-        answer
-):
-    res = openai.ChatCompletion.create(
-        model="gpt-3.5-turbo",
-        max_tokens=int(max_tokens),
-        temperature=float(temperature),
-        top_p=float(top_p),
-        frequency_penalty=float(frequency_penalty),
-        messages=[
-            {
-                "role": "system",
-                "content": "You are a IELTS examiner.",
-            },
-            {
-                "role": "system",
-                "content": f"The question you have to grade is of type {question_type} and is the following: {question}",
-            },
-            {
-                "role": "system",
-                "content": "Please provide a JSON object response with the overall grade and breakdown grades, "
-                           "formatted as follows: {'overall': 7.0, 'task_response': {'Task Achievement': 8.0, "
-                           "'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, 'Grammatical Range and Accuracy': "
-                           "6.0}}",
-            },
-            {
-                "role": "system",
-                "content": "Don't give explanations for the grades, just provide the json with the grades.",
-            },
-            {
-                "role": "user",
-                "content": f"Evaluate this answer according to ielts grading system: {answer}",
-            },
-        ],
-    )
-    return res["choices"][0]["message"]["content"]
-
-
-# Set the application title
-st.title("GPT-3.5 IELTS Examiner")
-
-# qt_col, q_col = st.columns(2)
-
-# Selection box to select the question type
-# with qt_col:
-question_type = st.selectbox(
-    "What is the question type?",
-    (
-        "Writing Task 2",
-    ),
-)
-
-# Provide the input area for question to be answered
-# with q_col:
-question = st.text_area("Enter the question:", height=100)
-
-# Provide the input area for text to be summarized
-answer = st.text_area("Enter the answer:", height=100)
-
-# Initiate two columns for section to be side-by-side
-# col1, col2 = st.columns(2)
-
-# Slider to control the model hyperparameter
-# with col1:
-token = st.slider("Token", min_value=0.0,
-                  max_value=2000.0, value=1000.0, step=1.0)
-temp = st.slider("Temperature", min_value=0.0,
-                 max_value=1.0, value=0.7, step=0.01)
-top_p = st.slider("Top_p", min_value=0.0, max_value=1.0, value=0.9, step=0.01)
-f_pen = st.slider("Frequency Penalty", min_value=-1.0,
-                  max_value=1.0, value=0.5, step=0.01)
-
-# Showing the current parameter used for the model
-# with col2:
-with st.expander("Current Parameter"):
-    st.write("Current Token :", token)
-    st.write("Current Temperature :", temp)
-    st.write("Current Nucleus Sampling :", top_p)
-    st.write("Current Frequency Penalty :", f_pen)
-
-# Creating button for execute the text summarization
-if st.button("Grade"):
-    st.write(generate_summarizer(token, temp, top_p,
-             f_pen, question_type, question, answer))