Merged develop into feature/training-content

And this is why llm code shouldn't be copy pasted blindly

Approved-by: Tiago Ribeiro

.dockerignore

@@ -5,3 +5,4 @@ README.md
 *.pyd
 __pycache__
 .pytest_cache
+/scripts

.env

@@ -1,5 +0,0 @@
-OPENAI_API_KEY=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
-JWT_SECRET_KEY=6e9c124ba92e8814719dcb0f21200c8aa4d0f119a994ac5e06eb90a366c83ab2
-JWT_TEST_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0In0.Emrs2D3BmMP4b3zMjw0fJTPeyMwWEBDbxx2vvaWguO0
-GOOGLE_APPLICATION_CREDENTIALS=firebase-configs/storied-phalanx-349916.json
-HEY_GEN_TOKEN=MjY4MDE0MjdjZmNhNDFmYTlhZGRkNmI3MGFlMzYwZDItMTY5NTExNzY3MA==

.gitignore

@@ -1,3 +1,6 @@
 __pycache__
 .idea
-.env
+.env
+.DS_Store
+/firebase-configs/test_firebase.json
+/scripts

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/ielts-be.iml

@@ -1,24 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="PYTHON_MODULE" version="4">
-  <component name="Flask">
-    <option name="enabled" value="true" />
-  </component>
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$">
-      <excludeFolder url="file://$MODULE_DIR$/venv" />
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.9" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.11 (ielts-be)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PackageRequirementsSettings">
     <option name="versionSpecifier" value="Don't specify version" />
   </component>
-  <component name="TemplatesService">
+  <component name="PyDocumentationSettings">
-    <option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
+    <option name="format" value="GOOGLE" />
-    <option name="TEMPLATE_FOLDERS">
+    <option name="myDocStringFormat" value="Google" />
-      <list>
-        <option value="$MODULE_DIR$/../flaskProject\templates" />
-      </list>
-    </option>
   </component>
 </module>

.idea/misc.xml

@@ -1,4 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" />
+  <component name="Black">
+    <option name="sdkName" value="Python 3.11 (ielts-be)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11 (ielts-be)" project-jdk-type="Python SDK" />
+  <component name="PyCharmProfessionalAdvertiser">
+    <option name="shown" value="true" />
+  </component>
 </project>

.idea/vcs.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+    <mapping directory="" vcs="Git" />
   </component>
 </project>

Dockerfile

@@ -11,7 +11,23 @@ ENV APP_HOME /app
 WORKDIR $APP_HOME
 COPY . ./
 
-RUN apt update && apt install -y ffmpeg
+RUN apt update && apt install -y \
+    ffmpeg \
+    poppler-utils \
+    texlive-latex-base \
+    texlive-fonts-recommended \
+    texlive-latex-extra \
+    texlive-xetex \
+    pandoc \
+    librsvg2-bin \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+
+RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs
+
+RUN npm install -g firebase-tools
 
 # Install production dependencies.
 RUN pip install --no-cache-dir -r requirements.txt

audio-samples/placeholder.txt

@@ -0,0 +1 @@
+THIS FILE ONLY EXISTS TO KEEP THIS FOLDER IN THE REPO

firebase-configs/encoach-staging.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "encoach-staging",
+  "private_key_id": "5718a649419776df9637589f8696a258a6a70f6c",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC2C6Es2gY8lLvH\ndVilNtRNm9glSaPXMNw2PzZZbSGuG1uGPFaCzlq1lOb2u17YfMG4GriKIMjIQKXF\nqdvxA8CAmAFRuDjUGmpbO/X1ZW7amOs5Bjed2BYmL01dEqzzwwh7rEfNDjeghRPx\n1uKzH8A6TLT5xq+74I5K1CIgiljBpZimsERu2SDawjkdtZfA7qoylA46Nq66LuwQ\nVyv9CK2SZNpBcT3sunCmRsrCzmSTzKdbcqRPdqUKgZOH/Rjp0sw9VuUgwoxdGZV3\n5SJjObo5ceZ1OSiJm7GwLzp7uq16sqycgSYwppNLI5OtzOfSuWbGD4+a044t2Mlq\n9PHXv7H/AgMBAAECggEAAfhKlFwq8MaL6PggRJq9HbaKgQ4fcOmCmy8AQmPNF1UM\nyVKSKGndjxUfPLCWsaaunUnjZlHoKkvndKXxDyttuVaBE9EiWEqNjRLZ3KpuJ9Jm\nH+CtLbmUCnISQb1n1AlvvZAwhLZbLBL/PhYyWiLapybZAdJAaOWLVKGgBD8gVRQW\nJFCqnszX1O2YlpWHutb979R4qoY/XAf94gyMkTpXZwuETvFqZbau2vxRZ8qARix3\nmic881PwiF6Cod8UPCS9yMK+Q+Se6SomwXU9PCmlummn9xmQBAxYy8gIAVs/J9Fg\n5SvhnImAPDd+zIzzw2cHCiruNWIhroMVZDZJgWdY1QKBgQDjTKKeFOur3ijJJL2/\nWg1SE2jLP0GpXzM5YMx6jdOCNDCzugPngRucRXiTkJ2FnUgyMcQyi6hyrbWXN/6z\nXhx5fwLB4tnTcqOMvNfcay5mDk3RW9ZZJxayB54Sf1Nm/4xiDBnGPT+iHQvK+/pT\nwScWznFkmk60E796o76OLn3PEwKBgQDNCC2uPq+uOcCopIO8HH88gqdxTvpbeHUU\nrdJOmr1VtGNuvay/mfpva9+VEtGbZTFzjhfvfCEIjpj3Llh8Flb9EYa6BmscBiyp\ngszEeFuB3zHndlSCZPnGJ7JiRAdPAEgG3Gl/r9th6PDaEMq0MFS5i7GGhPBIRYCG\nUtmY5eVy5QKBgH5Nuls/YsnJFD7ZNLscziQadvPhvZnhNbSfjmBXaP2EBMAKEFtX\nCcGndN4C0RVLFbAWqWAw7LR0xGA4FEcVd5snsZ+Nb98oZ6sv0H9B67F4J1O7xXsa\n1mitBPBgYjbsr9RXxwa6SB7MJx5vMGXUAeWRZ78wY6V7B76dOKkHOo+TAoGBAJf5\nBOsPueZZFm2qK58GPGVcrsI0+StNuPLP+H+dANQC9mTCIMaQWmm2Oq5jmYwmUKZH\nX4R6rH2MPOOSrbGkWWwRTpyaX1ARX49xzVefoqw8BOB8/Bz+vYjcKcPeitBK9Bhp\nzaUAc4s6PzRTl/xBirtRSQ/df8ECC0cFKBbF6PHlAoGAGqnlpo+k8vAtg6ulCuGu\nx2Y/c5UmvXGHk60pccnW3UtENSDnl99OgMfBz8/qLAMWs6DUQ/kvSlHQPmMBHRWZ\nNTr6ceGXyNs4KdYoj1K7AU3c0Lm0wyQ2giQMoOOUQAm98Xr8z5aiihj10hHPmzzL\n9kwpOmZpjNmC/ERD69imWhY=\n-----END PRIVATE KEY-----\n",
+  "client_email": "firebase-adminsdk-8rs9e@encoach-staging.iam.gserviceaccount.com",
+  "client_id": "108221424237414412378",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-8rs9e%40encoach-staging.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

generate_speaking_questions.py

@@ -1,180 +0,0 @@
-import random
-
-from helper.firebase_helper import upload_file_firebase_get_url
-from helper.heygen_api import create_video
-import os
-import uuid
-
-import firebase_admin
-from firebase_admin import credentials, firestore
-
-from dotenv import load_dotenv
-
-from heygen.AvatarEnum import AvatarEnum
-
-load_dotenv()
-
-# Initialize Firebase Admin SDK
-cred = credentials.Certificate(os.getenv("GOOGLE_APPLICATION_CREDENTIALS"))
-
-firebase_admin.initialize_app(cred)
-
-FIREBASE_BUCKET = 'storied-phalanx-349916.appspot.com'
-VIDEO_FILES_PATH = 'download-video/'
-FIREBASE_SPEAKING_VIDEO_FILES_PATH = 'speaking_videos/'
-
-# PART 1
-sp1_video_url = ""
-sp1_video_path = ""
-
-# sp1_questions_json = {
-#   "topic": "Hobbies and Interests",
-#   "question": "What do you like to do in your free time? Do you have any hobbies or interests? Are there any new hobbies you would like to try in the future?"
-# }
-sp1_questions_json = {
-    "topic": "Hobbies and Interests",
-    "question": "Do you have any hobbies or interests that you enjoy in your free time? Please describe one of your favorite hobbies and explain why you like it."
-}
-
-sp1_result = create_video(sp1_questions_json["question"], random.choice(list(AvatarEnum)))
-if sp1_result is not None:
-    sound_file_path = VIDEO_FILES_PATH + sp1_result
-    firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp1_result
-    url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
-    sp1_video_path = firebase_file_path
-    sp1_video_url = url
-else:
-    print("Failed to create video for question: " + sp1_questions_json["question"])
-
-# PART 2
-sp2_video_url = ""
-sp2_video_path = ""
-
-# sp2_questions_json = {
-#   "topic": "New Environment",
-#   "question": "Describe an occasion when you had to adapt to a new environment."
-# }
-
-sp2_questions_json = {
-    "topic": "Travel and Adventure",
-    "question": "Describe an occasion when you had to adapt to a new environment.",
-    "prompts": [
-        "Please include details about the destination",
-        "What you did there",
-        "why this trip was memorable for you."
-    ]
-}
-
-sp2_result = create_video(sp2_questions_json["question"], random.choice(list(AvatarEnum)))
-if sp2_result is not None:
-    sound_file_path = VIDEO_FILES_PATH + sp2_result
-    firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp2_result
-    url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
-    sp2_video_path = firebase_file_path
-    sp2_video_url = url
-else:
-    print("Failed to create video for question: " + sp2_questions_json["question"])
-
-# PART 3
-# 1
-# sp3_questions_json = {
-#     "topic": "Technology and Society",
-#     "questions": [
-#         "How do you think technology has affected the way people communicate with each other in today's society?",
-#         "In what ways has the use of smartphones and social media platforms changed the dynamics of personal relationships?",
-#         "Some argue that technology has made communication more convenient, while others worry that it has led to a decline in face-to-face interactions. What's your perspective on this matter, and how do you think it might impact future generations?"
-#     ]
-# }
-
-# 2
-# sp3_questions_json = {
-#     "topic": "Environmental Concerns",
-#     "questions": [
-#         "What do you believe are the most pressing environmental issues facing the world today?",
-#         "In your opinion, what role should governments play in addressing environmental concerns?",
-#         "Many people argue that individual actions, like recycling and conserving energy, are not sufficient to address environmental issues. What's your perspective on this? How can individuals contribute to solving these problems?"
-#     ]
-# }
-
-# 3
-# sp3_questions_json = {
-#   "topic": "Education and Technology",
-#   "questions": [
-#     "How has technology impacted the way students learn in today's classrooms?",
-#     "Do you think traditional textbooks will become obsolete in the future due to digital learning materials? Why or why not?",
-#     "What are the potential advantages and disadvantages of online courses and e-learning platforms?"
-#   ]
-# }
-
-# 4
-sp3_questions_json = {
-  "topic": "Cultural Diversity",
-  "questions": [
-    "How can exposure to different cultures benefit individuals and society as a whole?",
-    "What challenges might arise in culturally diverse communities, and how can they be addressed?",
-    "Do you think it's important for schools to incorporate cultural diversity into their curriculum? Why or why not?"
-  ]
-}
-
-questions = []
-avatar = random.choice(list(AvatarEnum))
-for question in sp3_questions_json["questions"]:
-    result = create_video(question, avatar)
-    if result is not None:
-        sound_file_path = VIDEO_FILES_PATH + result
-        firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + result
-        url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
-        video = {
-            "text": question,
-            "video_path": firebase_file_path,
-            "video_url": url
-        }
-        questions.append(video)
-    else:
-        print("Failed to create video for question: " + question)
-
-# BUILD OBJECT TO SAVE
-if len(questions) == len(sp3_questions_json["questions"]):
-    speaking_module_to_insert = {
-        "exercises": [
-            {
-                "id": str(uuid.uuid4()),
-                "prompts": [],
-                "text": sp1_questions_json["question"],
-                "title": sp1_questions_json["topic"],
-                "video_url": sp1_video_url,
-                "video_path": sp1_video_path,
-                "type": "speaking"
-            },
-            {
-                "id": str(uuid.uuid4()),
-                "prompts": sp2_questions_json["prompts"],
-                "text": sp2_questions_json["question"],
-                "title": sp2_questions_json["topic"],
-                "video_url": sp2_video_url,
-                "video_path": sp2_video_path,
-                "type": "speaking"
-            },
-            {
-                "id": str(uuid.uuid4()),
-                "prompts": questions,
-                "text": "Listen carefully and respond.",
-                "title": sp3_questions_json["topic"],
-                "type": "interactiveSpeaking"
-            }
-        ],
-        "isDiagnostic": True,
-        "minTimer": 14,
-        "module": "speaking"
-    }
-
-    db = firestore.client()
-
-    # JSON data to insert
-
-    # Add the JSON data to Firestore
-    collection_ref = db.collection('speaking')
-    document_ref = collection_ref.add(speaking_module_to_insert)
-    print(f"Document added with ID: {document_ref}")
-else:
-    print("Array sizes do not match. Video uploading failing is probably the cause.")

helper/api_messages.py

@@ -16,9 +16,6 @@ class QuestionType(Enum):
     READING_PASSAGE_2 = "Reading Passage 2"
     READING_PASSAGE_3 = "Reading Passage 3"
 
-class ExerciseType(Enum):
-    MULTIPLE_CHOICE = "multiple choice"
-
 
 def get_grading_messages(question_type: QuestionType, question: str, answer: str, context: str = None):
     if QuestionType.WRITING_TASK_1 == question_type:

helper/constants.py

@@ -1,4 +1,3 @@
-FIREBASE_BUCKET = 'storied-phalanx-349916.appspot.com'
 AUDIO_FILES_PATH = 'download-audio/'
 FIREBASE_LISTENING_AUDIO_FILES_PATH = 'listening_recordings/'
 VIDEO_FILES_PATH = 'download-video/'
@@ -8,15 +7,24 @@ GRADING_TEMPERATURE = 0.1
 TIPS_TEMPERATURE = 0.2
 GEN_QUESTION_TEMPERATURE = 0.7
 GPT_3_5_TURBO = "gpt-3.5-turbo"
+GPT_4_TURBO = "gpt-4-turbo"
+GPT_4_O = "gpt-4o"
 GPT_3_5_TURBO_16K = "gpt-3.5-turbo-16k"
 GPT_3_5_TURBO_INSTRUCT = "gpt-3.5-turbo-instruct"
+GPT_4_PREVIEW = "gpt-4-turbo-preview"
 
 GRADING_FIELDS = ['comment', 'overall', 'task_response']
 GEN_FIELDS = ['topic']
 GEN_TEXT_FIELDS = ['title']
 LISTENING_GEN_FIELDS = ['transcript', 'exercise']
-READING_EXERCISE_TYPES = ['multipleChoice', 'fillBlanks', 'writeBlanks', 'trueFalse']
+READING_EXERCISE_TYPES = ['fillBlanks', 'writeBlanks', 'trueFalse', 'paragraphMatch']
+READING_3_EXERCISE_TYPES = ['fillBlanks', 'writeBlanks', 'trueFalse', 'paragraphMatch', 'ideaMatch']
 LISTENING_EXERCISE_TYPES = ['multipleChoice', 'writeBlanksQuestions', 'writeBlanksFill', 'writeBlanksForm']
+LISTENING_1_EXERCISE_TYPES = ['multipleChoice', 'writeBlanksQuestions', 'writeBlanksFill', 'writeBlanksFill',
+                              'writeBlanksForm', 'writeBlanksForm', 'writeBlanksForm', 'writeBlanksForm']
+LISTENING_2_EXERCISE_TYPES = ['multipleChoice', 'writeBlanksQuestions']
+LISTENING_3_EXERCISE_TYPES = ['multipleChoice3Options', 'writeBlanksQuestions']
+LISTENING_4_EXERCISE_TYPES = ['multipleChoice', 'writeBlanksQuestions', 'writeBlanksFill', 'writeBlanksForm']
 
 TOTAL_READING_PASSAGE_1_EXERCISES = 13
 TOTAL_READING_PASSAGE_2_EXERCISES = 13
@@ -32,9 +40,9 @@ WRITING_MIN_TIMER_DEFAULT = 60
 SPEAKING_MIN_TIMER_DEFAULT = 14
 
 BLACKLISTED_WORDS = ["jesus", "sex", "gay", "lesbian", "homosexual", "god", "angel", "pornography", "beer", "wine",
-                     "cocaine", "drugs", "alcohol", "nudity", "lgbt", "casino", "gambling", "gaming", "catholicism",
+                     "cocaine", "alcohol", "nudity", "lgbt", "casino", "gambling", "catholicism",
-                     "discrimination", "politics", "politic", "christianity", "islam", "christian", "christians",
+                     "discrimination", "politic", "christianity", "islam", "christian", "christians",
-                     "jews", "jew", "policies", "human rights", "discrimination", "discriminatory"]
+                     "jews", "jew", "discrimination", "discriminatory"]
 
 EN_US_VOICES = [
     {'Gender': 'Female', 'Id': 'Salli', 'LanguageCode': 'en-US', 'LanguageName': 'US English', 'Name': 'Salli',
@@ -108,6 +116,57 @@ FEMALE_NEURAL_VOICES = [item for item in ALL_NEURAL_VOICES if item.get('Gender')
 
 difficulties = ["easy", "medium", "hard"]
 
+mti_topics = [
+    "Education",
+    "Technology",
+    "Environment",
+    "Health and Fitness",
+    "Engineering",
+    "Work and Careers",
+    "Travel and Tourism",
+    "Culture and Traditions",
+    "Social Issues",
+    "Arts and Entertainment",
+    "Climate Change",
+    "Social Media",
+    "Sustainable Development",
+    "Health Care",
+    "Immigration",
+    "Artificial Intelligence",
+    "Consumerism",
+    "Online Shopping",
+    "Energy",
+    "Oil and Gas",
+    "Poverty and Inequality",
+    "Cultural Diversity",
+    "Democracy and Governance",
+    "Mental Health",
+    "Ethics and Morality",
+    "Population Growth",
+    "Science and Innovation",
+    "Poverty Alleviation",
+    "Cybersecurity and Privacy",
+    "Human Rights",
+    "Food and Agriculture",
+    "Cyberbullying and Online Safety",
+    "Linguistic Diversity",
+    "Urbanization",
+    "Artificial Intelligence in Education",
+    "Youth Empowerment",
+    "Disaster Management",
+    "Mental Health Stigma",
+    "Internet Censorship",
+    "Sustainable Fashion",
+    "Indigenous Rights",
+    "Water Scarcity",
+    "Social Entrepreneurship",
+    "Privacy in the Digital Age",
+    "Sustainable Transportation",
+    "Gender Equality",
+    "Automation and Job Displacement",
+    "Digital Divide",
+    "Education Inequality"
+]
 topics = [
     "Art and Creativity",
     "History of Ancient Civilizations",
@@ -115,13 +174,11 @@ topics = [
     "Space Exploration",
     "Artificial Intelligence",
     "Climate Change",
-    "World Religions",
     "The Human Brain",
     "Renewable Energy",
     "Cultural Diversity",
     "Modern Technology Trends",
     "Sustainable Agriculture",
-    "Globalization",
     "Natural Disasters",
     "Cybersecurity",
     "Philosophy of Ethics",
@@ -129,7 +186,6 @@ topics = [
     "Health and Wellness",
     "Literature and Classics",
     "World Geography",
-    "Music and Its Influence",
     "Social Media Impact",
     "Food Sustainability",
     "Economics and Markets",
@@ -149,7 +205,6 @@ topics = [
     "Renewable Energy Sources",
     "Cultural Heritage Preservation",
     "Modern Art Movements",
-    "Immigration Issues",
     "Sustainable Transportation",
     "The History of Medicine",
     "Artificial Neural Networks",
@@ -161,7 +216,6 @@ topics = [
     "World Oceans",
     "Social Networking",
     "Sustainable Fashion",
-    "International Trade",
     "Prehistoric Era",
     "Democracy and Governance",
     "Postcolonial Literature",
@@ -172,21 +226,17 @@ topics = [
     "Education Technology",
     "Renaissance Art",
     "Renewable Energy Policy",
-    "Cultural Festivals",
     "Modern Architecture",
     "Climate Resilience",
     "Artificial Life",
     "Fitness and Nutrition",
     "Classic Literature Adaptations",
-    "World History Wars",
     "Ethical Dilemmas",
     "Internet of Things (IoT)",
     "Meditation Practices",
     "Literary Symbolism",
     "Marine Conservation",
-    "Social Justice Movements",
     "Sustainable Tourism",
-    "International Finance",
     "Ancient Philosophy",
     "Cold War Era",
     "Behavioral Economics",
@@ -195,11 +245,9 @@ topics = [
     "Cultural Exchange",
     "Modern Sculpture",
     "Climate Mitigation",
-    "Artificial Intelligence Ethics",
     "Mindfulness",
     "Literary Criticism",
     "Wildlife Conservation",
-    "Political Activism",
     "Renewable Energy Innovations",
     "History of Mathematics",
     "Human-Computer Interaction",
@@ -207,13 +255,11 @@ topics = [
     "Cultural Appropriation",
     "Traditional cuisine and culinary arts",
     "Local music and dance traditions",
-    "Cultural festivals and celebrations",
     "History of the region and historical landmarks",
     "Traditional crafts and artisanal skills",
     "Wildlife and conservation efforts",
     "Local sports and athletic competitions",
     "Fashion trends and clothing styles",
-    "Literature and poetry from the region",
     "Education systems and advancements",
     "Healthcare services and medical innovations",
     "Family values and social dynamics",
@@ -223,8 +269,7 @@ topics = [
     "Entrepreneurship and business ventures",
     "Youth empowerment initiatives",
     "Art exhibitions and cultural events",
-    "Philanthropy and community development projects",
+    "Philanthropy and community development projects"
-    "Political developments and civic engagement efforts"
 ]
 
 two_people_scenarios = [
@@ -394,7 +439,6 @@ social_monologue_contexts = [
     "A monologue about the impact of technological advancements",
     "An explanation of the process of wildlife rehabilitation",
     "A presentation on the history of a famous explorer",
-    "An overview of traditional storytelling from different cultures",
     "A lecture on the principles of effective marketing",
     "A discussion about the challenges of environmental sustainability",
     "A monologue about the influence of social entrepreneurship",
@@ -615,5 +659,3 @@ academic_subjects = [
     "Ecology",
     "International Business"
 ]
-
-

helper/exam_variant.py

@@ -3,4 +3,4 @@ from enum import Enum
 
 class ExamVariant(Enum):
     FULL = "full"
-    PARTIAL = "partial"
+    PARTIAL = "partial"

helper/file_helper.py

@@ -1,5 +1,5 @@
-import os
 import datetime
+import os
 from pathlib import Path
 
 
@@ -12,6 +12,6 @@ def delete_files_older_than_one_day(directory):
             file_name = file_path.name
             file_modified_time = datetime.datetime.fromtimestamp(file_path.stat().st_mtime)
             time_difference = current_time - file_modified_time
-            if time_difference.days > 1  and "placeholder" not in file_name:
+            if time_difference.days > 1 and "placeholder" not in file_name:
                 file_path.unlink()
-                print(f"Deleted file: {file_path}")
+                print(f"Deleted file: {file_path}")

helper/firebase_helper.py

@@ -1,15 +1,7 @@
 import logging
 
 from google.cloud import storage
-import os
+from pymongo.database import Database
-import uuid
-
-import firebase_admin
-from firebase_admin import credentials, firestore
-
-from dotenv import load_dotenv
-
-import app
 
 
 def download_firebase_file(bucket_name, source_blob_name, destination_file_name):
@@ -18,9 +10,10 @@ def download_firebase_file(bucket_name, source_blob_name, destination_file_name)
     bucket = storage_client.bucket(bucket_name)
     blob = bucket.blob(source_blob_name)
     blob.download_to_filename(destination_file_name)
-    logging.info(f"File uploaded to {destination_file_name}")
+    logging.info(f"File downloaded to {destination_file_name}")
     return destination_file_name
 
+
 def upload_file_firebase(bucket_name, destination_blob_name, source_file_name):
     # Uploads a file to Firebase Storage.
     storage_client = storage.Client()
@@ -31,9 +24,11 @@ def upload_file_firebase(bucket_name, destination_blob_name, source_file_name):
         logging.info(f"File uploaded to {destination_blob_name}")
         return True
     except Exception as e:
+        import app
         app.app.logger.error("Error uploading file to Google Cloud Storage: " + str(e))
         return False
 
+
 def upload_file_firebase_get_url(bucket_name, destination_blob_name, source_file_name):
     # Uploads a file to Firebase Storage.
     storage_client = storage.Client()
@@ -50,41 +45,21 @@ def upload_file_firebase_get_url(bucket_name, destination_blob_name, source_file
         url = blob.public_url
         return url
     except Exception as e:
+        import app
         app.app.logger.error("Error uploading file to Google Cloud Storage: " + str(e))
         return None
 
-def save_to_db(collection: str, item):
+
-    db = firestore.client()
+def save_to_db_with_id(mongo_db: Database, collection: str, item, id: str):
-    collection_ref = db.collection(collection)
+    collection_ref = mongo_db[collection]
-    (update_time, document_ref) = collection_ref.add(item)
+
+    document_ref = collection_ref.insert_one({"id": id, **item})
     if document_ref:
-        logging.info(f"Document added with ID: {document_ref.id}")
+        logging.info(f"Document added with ID: {document_ref.inserted_id}")
-        return (True, document_ref.id)
+        return (True, document_ref.inserted_id)
     else:
         return (False, None)
 
 
-def save_to_db_with_id(collection: str, item, id: str):
+def get_all(mongo_db: Database, collection: str):
-    db = firestore.client()
+    return list(mongo_db[collection].find())
-    collection_ref = db.collection(collection)
-    # Reference to the specific document with the desired ID
-    document_ref = collection_ref.document(id)
-    # Set the data to the document
-    document_ref.set(item)
-    if document_ref:
-        logging.info(f"Document added with ID: {document_ref.id}")
-        return (True, document_ref.id)
-    else:
-        return (False, None)
-
-def get_all(collection: str):
-    db = firestore.client()
-    collection_ref = db.collection(collection)
-
-    all_exercises = (
-        collection_ref
-        .get()
-    )
-
-    return all_exercises
-

helper/generate_jwt.py

@@ -1,6 +1,6 @@
-import jwt
 import os
 
+import jwt
 from dotenv import load_dotenv
 
 load_dotenv()

helper/gpt_zero.py

@@ -0,0 +1,50 @@
+from logging import getLogger
+from typing import Dict, Optional
+import requests
+
+
+class GPTZero:
+    _GPT_ZERO_ENDPOINT = 'https://api.gptzero.me/v2/predict/text'
+
+    def __init__(self, gpt_zero_key: str):
+        self._logger = getLogger(__name__)
+        if gpt_zero_key is None:
+            self._logger.warning('GPT Zero key was not included! Skipping ai detection when grading.')
+        self._gpt_zero_key = gpt_zero_key
+        self._header = {
+            'x-api-key': gpt_zero_key
+        }
+
+    def run_detection(self, text: str):
+        if self._gpt_zero_key is None:
+            return None
+        data = {
+            'document': text,
+            'version': '',
+            'multilingual': False
+        }
+        response = requests.post(self._GPT_ZERO_ENDPOINT, headers=self._header, json=data)
+        if response.status_code != 200:
+            self._logger.error(f'GPT\'s Zero Endpoint returned with {response.status_code}: {response.json()}')
+            return None
+        return self._parse_detection(response.json())
+
+    def _parse_detection(self, response: Dict) -> Optional[Dict]:
+        try:
+            text_scan = response["documents"][0]
+            filtered_sentences = [
+                {
+                    "sentence": item["sentence"],
+                    "highlight_sentence_for_ai": item["highlight_sentence_for_ai"]
+                }
+                for item in text_scan["sentences"]
+            ]
+            return {
+                "class_probabilities": text_scan["class_probabilities"],
+                "confidence_category": text_scan["confidence_category"],
+                "predicted_class": text_scan["predicted_class"],
+                "sentences": filtered_sentences
+            }
+        except Exception as e:
+            self._logger.error(f'Failed to parse GPT\'s Zero response: {str(e)}')
+            return None

helper/heygen_api.py

@@ -1,21 +1,22 @@
-import logging
 import os
 import random
+import time
+from logging import getLogger
 
 import requests
-import time
-
 from dotenv import load_dotenv
 
-import app
 from helper.constants import *
 from helper.firebase_helper import upload_file_firebase_get_url, save_to_db_with_id
 from heygen.AvatarEnum import AvatarEnum
 
 load_dotenv()
 
+logger = getLogger(__name__)
+
 # Get HeyGen token
 TOKEN = os.getenv("HEY_GEN_TOKEN")
+FIREBASE_BUCKET = os.getenv('FIREBASE_BUCKET')
 
 # POST TO CREATE VIDEO
 CREATE_VIDEO_URL = 'https://api.heygen.com/v1/template.generate'
@@ -28,36 +29,34 @@ GET_HEADER = {
     'X-Api-Key': TOKEN
 }
 
-MATTHEW_NOAH = "11b234e504e44bfda9bc6b7aac3c8f81"
-VERA_CERISE = "9bf2f27009cd403ab4ba4e22629b27bb"
-EDWARD_TONY = "d3333e37952946059b45efc8482a2b6c"
-TANYA_MOLLY = "07c75076b3f94df4ac658c6de72be83a"
-KAYLA_ABBI = "d688099f8db9472cb4890b0561e81793"
-JEROME_RYAN = "ad41feb2a5c4483085525e3d8907f512"
-TYLER_CHRISTOPHER = "03c796f8ed274bb38f19e893bcbc6121"
-
 
 def create_videos_and_save_to_db(exercises, template, id):
+    avatar = random.choice(list(AvatarEnum))
     # Speaking 1
     # Using list comprehension to find the element with the desired value in the 'type' field
     found_exercises_1 = [element for element in exercises if element.get('type') == 1]
     # Check if any elements were found
     if found_exercises_1:
         exercise_1 = found_exercises_1[0]
-        app.app.logger.info('Creating video for speaking part 1')
+        sp1_questions = []
-        sp1_result = create_video(exercise_1["question"], random.choice(list(AvatarEnum)))
+        logger.info('Creating video for speaking part 1')
-        if sp1_result is not None:
+        for question in exercise_1["questions"]:
-            sound_file_path = VIDEO_FILES_PATH + sp1_result
+            sp1_result = create_video(question, avatar)
-            firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp1_result
+            if sp1_result is not None:
-            url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
+                sound_file_path = VIDEO_FILES_PATH + sp1_result
-            sp1_video_path = firebase_file_path
+                firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp1_result
-            sp1_video_url = url
+                url = upload_file_firebase_get_url(FIREBASE_BUCKET, firebase_file_path, sound_file_path)
-            template["exercises"][0]["text"] = exercise_1["question"]
+                video = {
-            template["exercises"][0]["title"] = exercise_1["topic"]
+                    "text": question,
-            template["exercises"][0]["video_url"] = sp1_video_url
+                    "video_path": firebase_file_path,
-            template["exercises"][0]["video_path"] = sp1_video_path
+                    "video_url": url
-        else:
+                }
-            app.app.logger.error("Failed to create video for part 1 question: " + exercise_1["question"])
+                sp1_questions.append(video)
+            else:
+                logger.error("Failed to create video for part 1 question: " + exercise_1["question"])
+        template["exercises"][0]["prompts"] = sp1_questions
+        template["exercises"][0]["first_title"] = exercise_1["first_topic"]
+        template["exercises"][0]["second_title"] = exercise_1["second_topic"]
 
     # Speaking 2
     # Using list comprehension to find the element with the desired value in the 'type' field
@@ -65,8 +64,8 @@ def create_videos_and_save_to_db(exercises, template, id):
     # Check if any elements were found
     if found_exercises_2:
         exercise_2 = found_exercises_2[0]
-        app.app.logger.info('Creating video for speaking part 2')
+        logger.info('Creating video for speaking part 2')
-        sp2_result = create_video(exercise_2["question"], random.choice(list(AvatarEnum)))
+        sp2_result = create_video(exercise_2["question"], avatar)
         if sp2_result is not None:
             sound_file_path = VIDEO_FILES_PATH + sp2_result
             firebase_file_path = FIREBASE_SPEAKING_VIDEO_FILES_PATH + sp2_result
@@ -79,7 +78,7 @@ def create_videos_and_save_to_db(exercises, template, id):
             template["exercises"][1]["video_url"] = sp2_video_url
             template["exercises"][1]["video_path"] = sp2_video_path
         else:
-            app.app.logger.error("Failed to create video for part 2 question: " + exercise_2["question"])
+            logger.error("Failed to create video for part 2 question: " + exercise_2["question"])
 
     # Speaking 3
     # Using list comprehension to find the element with the desired value in the 'type' field
@@ -88,8 +87,7 @@ def create_videos_and_save_to_db(exercises, template, id):
     if found_exercises_3:
         exercise_3 = found_exercises_3[0]
         sp3_questions = []
-        avatar = random.choice(list(AvatarEnum))
+        logger.info('Creating videos for speaking part 3')
-        app.app.logger.info('Creating videos for speaking part 3')
         for question in exercise_3["questions"]:
             result = create_video(question, avatar)
             if result is not None:
@@ -103,7 +101,7 @@ def create_videos_and_save_to_db(exercises, template, id):
                 }
                 sp3_questions.append(video)
             else:
-                app.app.logger.error("Failed to create video for part 3 question: " + question)
+                logger.error("Failed to create video for part 3 question: " + question)
         template["exercises"][2]["prompts"] = sp3_questions
         template["exercises"][2]["title"] = exercise_3["topic"]
 
@@ -115,7 +113,8 @@ def create_videos_and_save_to_db(exercises, template, id):
         template["exercises"].pop(0)
 
     save_to_db_with_id("speaking", template, id)
-    app.app.logger.info('Saved speaking to DB with id ' + id + " : " + str(template))
+    logger.info('Saved speaking to DB with id ' + id + " : " + str(template))
+
 
 def create_video(text, avatar):
     # POST TO CREATE VIDEO
@@ -135,8 +134,8 @@ def create_video(text, avatar):
         }
     }
     response = requests.post(create_video_url, headers=POST_HEADER, json=data)
-    app.app.logger.info(response.status_code)
+    logger.info(response.status_code)
-    app.app.logger.info(response.json())
+    logger.info(response.json())
 
     # GET TO CHECK STATUS AND GET VIDEO WHEN READY
     video_id = response.json()["data"]["video_id"]
@@ -155,11 +154,11 @@ def create_video(text, avatar):
         error = response_data["data"]["error"]
 
         if status != "completed" and error is None:
-            app.app.logger.info(f"Status: {status}")
+            logger.info(f"Status: {status}")
             time.sleep(10)  # Wait for 10 second before the next request
 
-    app.app.logger.info(response.status_code)
+    logger.info(response.status_code)
-    app.app.logger.info(response.json())
+    logger.info(response.json())
 
     # DOWNLOAD VIDEO
     download_url = response.json()['data']['video_url']
@@ -173,8 +172,8 @@ def create_video(text, avatar):
         output_path = os.path.join(output_directory, output_filename)
         with open(output_path, 'wb') as f:
             f.write(response.content)
-        app.app.logger.info(f"File '{output_filename}' downloaded successfully.")
+        logger.info(f"File '{output_filename}' downloaded successfully.")
         return output_filename
     else:
-        app.app.logger.error(f"Failed to download file. Status code: {response.status_code}")
+        logger.error(f"Failed to download file. Status code: {response.status_code}")
         return None

helper/openai_interface.py

@@ -1,22 +1,21 @@
 import json
-import openai
 import os
 import re
 
 from dotenv import load_dotenv
+from openai import OpenAI
 
-from helper.constants import GPT_3_5_TURBO_INSTRUCT, BLACKLISTED_WORDS
+from helper.constants import BLACKLISTED_WORDS, GPT_3_5_TURBO
 from helper.token_counter import count_tokens
 
 load_dotenv()
-openai.api_key = os.getenv("OPENAI_API_KEY")
+client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))
 
 MAX_TOKENS = 4097
 TOP_P = 0.9
 FREQUENCY_PENALTY = 0.5
 
 TRY_LIMIT = 2
-
 try_count = 0
 
 # GRADING SUMMARY
@@ -40,6 +39,10 @@ tools = [{
                     "type": "string",
                     "description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.",
                 },
+                "bullet_points": {
+                    "type": "string",
+                    "description": "Text with four bullet points to improve the english speaking ability. Only include text for the bullet points separated by a paragraph. ",
+                },
             },
             "required": ["evaluation", "suggestions"],
         },
@@ -47,142 +50,44 @@ tools = [{
 }]
 
 
-###
-
-def process_response(input_string, quotation_check_field):
-    if '{' in input_string:
-        try:
-            # Find the index of the first occurrence of '{'
-            index = input_string.index('{')
-            # Extract everything after the first '{' (inclusive)
-            result = input_string[index:]
-            if re.search(r"'" + quotation_check_field + "':\s*'(.*?)'", result, re.DOTALL | re.MULTILINE) or \
-                    re.search(r"'" + quotation_check_field + "':\s*\[([^\]]+)]", result, re.DOTALL | re.MULTILINE):
-                json_obj = json.loads(parse_string(result))
-                return json_obj
-            else:
-                parsed_string = result.replace("\n\n", " ")
-                parsed_string = parsed_string.replace("\n", " ")
-                parsed_string = re.sub(r',\s*]', ']', parsed_string)
-                parsed_string = re.sub(r',\s*}', '}', parsed_string)
-                if (parsed_string.find('[') == -1) and (parsed_string.find(']') == -1):
-                    parsed_string = parse_string_2(parsed_string)
-                    return json.loads(parsed_string)
-
-                return json.loads(parsed_string)
-        except Exception as e:
-            print(f"Invalid JSON string! Exception: {e}")
-            print(f"String: {input_string}")
-            print(f"Exception: {e}")
-    else:
-        return input_string
-
-
-def parse_string(to_parse: str):
-    parsed_string = to_parse.replace("\"", "\\\"")
-    pattern = r"(?<!\w)'|'(?!\w)"
-    parsed_string = re.sub(pattern, '"', parsed_string)
-    parsed_string = parsed_string.replace("\\\"", "'")
-    parsed_string = parsed_string.replace("\n\n", " ")
-    parsed_string = re.sub(r',\s*]', ']', parsed_string)
-    parsed_string = re.sub(r',\s*}', '}', parsed_string)
-    return parsed_string
-
-
-def parse_string_2(to_parse: str):
-    keys_and_values_str = to_parse.replace("{", "").replace("}", "")
-    split_pattern = r'(?<="),|(?<="):'
-    keys_and_values = re.split(split_pattern, keys_and_values_str)
-
-    keys = []
-    values = []
-
-    for idx, x in enumerate(keys_and_values):
-        if (idx % 2) == 0:
-            keys.append(x)
-        else:
-            values.append(x)
-
-    parsed_values = []
-
-    for value in values:
-        parsed_values.append(("\"" + value.replace("\"", "").strip() + "\""))
-
-    for ind, parsed_value in enumerate(parsed_values):
-        to_parse = to_parse.replace(values[ind], parsed_values[ind])
-
-    to_parse = to_parse.replace(":", ": ")
-    return to_parse
-
-
-def remove_special_chars_and_escapes(input_string):
-    parsed_string = input_string.replace("\\\"", "'")
-    parsed_string = parsed_string.replace("\n\n", " ")
-    # Define a regular expression pattern to match special characters and escapes
-    pattern = r'(\\[nrt])|[^a-zA-Z0-9\s]'
-
-    # Use re.sub() to replace the matched patterns with an empty string
-    cleaned_string = re.sub(pattern, '', parsed_string)
-
-    return cleaned_string
-
-
 def check_fields(obj, fields):
     return all(field in obj for field in fields)
 
 
-def make_openai_call(model, messages, token_count, fields_to_check, temperature):
+def make_openai_call(model, messages, token_count, fields_to_check, temperature, check_blacklisted=True):
     global try_count
-    result = openai.ChatCompletion.create(
+    result = client.chat.completions.create(
         model=model,
         max_tokens=int(MAX_TOKENS - token_count - 300),
         temperature=float(temperature),
-        top_p=float(TOP_P),
+        messages=messages,
-        frequency_penalty=float(FREQUENCY_PENALTY),
+        response_format={"type": "json_object"}
-        messages=messages
     )
+    result = result.choices[0].message.content
+
+    if check_blacklisted:
+        found_blacklisted_word = get_found_blacklisted_words(result)
+
+        if found_blacklisted_word is not None and try_count < TRY_LIMIT:
+            from app import app
+            app.logger.warning("Result contains blacklisted words: " + str(found_blacklisted_word))
+            try_count = try_count + 1
+            return make_openai_call(model, messages, token_count, fields_to_check, temperature)
+        elif found_blacklisted_word is not None and try_count >= TRY_LIMIT:
+            return ""
 
     if fields_to_check is None:
-        return result["choices"][0]["message"]["content"]
+        return json.loads(result)
 
-    processed_response = process_response(result["choices"][0]["message"]["content"], fields_to_check[0])
+    if check_fields(result, fields_to_check) is False and try_count < TRY_LIMIT:
-
-    if check_fields(processed_response, fields_to_check) is False and try_count < TRY_LIMIT:
         try_count = try_count + 1
         return make_openai_call(model, messages, token_count, fields_to_check, temperature)
     elif try_count >= TRY_LIMIT:
         try_count = 0
-        return result["choices"][0]["message"]["content"]
+        return json.loads(result)
     else:
         try_count = 0
-        return processed_response
+        return json.loads(result)
-
-
-def make_openai_instruct_call(model, message: str, token_count, fields_to_check, temperature):
-    global try_count
-    response = openai.Completion.create(
-        model=model,
-        prompt=message,
-        max_tokens=int(4097 - token_count - 300),
-        temperature=0.7
-    )["choices"][0]["text"]
-
-    if has_blacklisted_words(response) and try_count < TRY_LIMIT:
-        try_count = try_count + 1
-        return make_openai_instruct_call(model, message, token_count, fields_to_check, temperature)
-    elif has_blacklisted_words(response) and try_count >= TRY_LIMIT:
-        return ""
-
-    if fields_to_check is None:
-        return response.replace("\n\n", " ").strip()
-
-    processed_response = process_response(response, fields_to_check[0])
-    if check_fields(processed_response, fields_to_check) is False and try_count < TRY_LIMIT:
-        try_count = try_count + 1
-        return make_openai_instruct_call(model, message, token_count, fields_to_check, temperature)
-    else:
-        try_count = 0
-        return processed_response
 
 
 # GRADING SUMMARY
@@ -193,9 +98,11 @@ def calculate_grading_summary(body):
 
     for section in extracted_sections:
         openai_response_dict = calculate_section_grade_summary(section)
+
         ret = ret + [{'code': section['code'], 'name': section['name'], 'grade': section['grade'],
                       'evaluation': openai_response_dict['evaluation'],
-                      'suggestions': openai_response_dict['suggestions']}]
+                      'suggestions': openai_response_dict['suggestions'],
+                      'bullet_points': parse_bullet_points(openai_response_dict['bullet_points'], section['grade'])}]
 
     return {'sections': ret}
 
@@ -215,7 +122,8 @@ def calculate_section_grade_summary(section):
          "content": "Don't offer suggestions in the evaluation comment. Only in the suggestions section."},
         {"role": "user",
          "content": "Your evaluation comment on the grade should enunciate the grade, be insightful, be speculative, be one paragraph long. "},
-        {"role": "user", "content": "Please save the evaluation comment and suggestions generated."}
+        {"role": "user", "content": "Please save the evaluation comment and suggestions generated."},
+        {"role": "user", "content": f"Offer bullet points to improve the english {str(section['name'])} ability."},
     ]
 
     if section['code'] == "level":
@@ -227,7 +135,7 @@ def calculate_section_grade_summary(section):
         messages[2:2] = [{"role": "user",
                           "content": "This section is s designed to assess the English language proficiency of individuals who want to study or work in English-speaking countries. The speaking section evaluates a candidate's ability to communicate effectively in spoken English."}]
 
-    res = openai.ChatCompletion.create(
+    res = client.chat.completions.create(
         model="gpt-3.5-turbo",
         max_tokens=chat_config['max_tokens'],
         temperature=chat_config['temperature'],
@@ -245,7 +153,7 @@ def parse_openai_response(response):
             response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
         return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
     else:
-        return {'evaluation': "", 'suggestions': ""}
+        return {'evaluation': "", 'suggestions': "", 'bullet_points': []}
 
 
 def extract_existing_sections_from_body(my_dict, keys_to_extract):
@@ -255,23 +163,88 @@ def extract_existing_sections_from_body(my_dict, keys_to_extract):
             my_dict['sections']))
 
 
+def parse_bullet_points(bullet_points_str, grade):
+    max_grade_for_suggestions = 9
+    if isinstance(bullet_points_str, str) and grade < max_grade_for_suggestions:
+        # Split the string by '\n'
+        lines = bullet_points_str.split('\n')
+
+        # Remove '-' and trim whitespace from each line
+        cleaned_lines = [line.replace('-', '').strip() for line in lines]
+
+        # Add '.' to lines that don't end with it
+        return [line + '.' if line and not line.endswith('.') else line for line in cleaned_lines]
+    else:
+        return []
+
+
 def get_fixed_text(text):
-    message = ('Fix the errors in the given text and put it in a JSON. Do not complete the answer, only replace what '
+    messages = [
-               'is wrong. Sample JSON: {"fixed_text": "fixed test with no '
+        {"role": "system", "content": ('You are a helpful assistant designed to output JSON on this format: '
-               'misspelling errors"}] \n The text: "' + text + '"')
+                                       '{"fixed_text": "fixed test with no misspelling errors"}')
-    token_count = count_tokens(message)["n_tokens"]
+         },
-    response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, ["fixed_text"], 0.2)
+        {"role": "user", "content": (
+                'Fix the errors in the given text and put it in a JSON. Do not complete the answer, only replace what '
+                'is wrong. \n The text: "' + text + '"')
+         }
+    ]
+    token_count = count_total_tokens(messages)
+    response = make_openai_call(GPT_3_5_TURBO, messages, token_count, ["fixed_text"], 0.2, False)
     return response["fixed_text"]
 
+
 def get_speaking_corrections(text):
-    message = ('Fix the errors in the provided transcription and put it in a JSON. Do not complete the answer, only '
+    messages = [
-               'replace what is wrong. Sample JSON: {"fixed_text": "fixed '
+        {"role": "system", "content": ('You are a helpful assistant designed to output JSON on this format: '
-               'transcription with no misspelling errors"}] \n The text: "' + text + '"')
+                                       '{"fixed_text": "fixed transcription with no misspelling errors"}')
-    token_count = count_tokens(message)["n_tokens"]
+         },
-    response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, ["fixed_text"], 0.2)
+        {"role": "user", "content": (
+                'Fix the errors in the provided transcription and put it in a JSON. Do not complete the answer, only '
+                'replace what is wrong. \n The text: "' + text + '"')
+         }
+    ]
+    token_count = count_total_tokens(messages)
+    response = make_openai_call(GPT_3_5_TURBO, messages, token_count, ["fixed_text"], 0.2, False)
     return response["fixed_text"]
 
 
 def has_blacklisted_words(text: str):
     text_lower = text.lower()
     return any(word in text_lower for word in BLACKLISTED_WORDS)
+
+
+def get_found_blacklisted_words(text: str):
+    text_lower = text.lower()
+    for word in BLACKLISTED_WORDS:
+        if re.search(r'\b' + re.escape(word) + r'\b', text_lower):
+            return word
+    return None
+
+
+def remove_special_characters_from_beginning(string):
+    cleaned_string = string.lstrip('\n')
+    if string.startswith("'") or string.startswith('"'):
+        cleaned_string = string[1:]
+    if cleaned_string.endswith('"'):
+        return cleaned_string[:-1]
+    else:
+        return cleaned_string
+
+
+def replace_expression_in_object(obj, expression, replacement):
+    if isinstance(obj, dict):
+        for key in obj:
+            if isinstance(obj[key], str):
+                obj[key] = obj[key].replace(expression, replacement)
+            elif isinstance(obj[key], list):
+                obj[key] = [replace_expression_in_object(item, expression, replacement) for item in obj[key]]
+            elif isinstance(obj[key], dict):
+                obj[key] = replace_expression_in_object(obj[key], expression, replacement)
+    return obj
+
+
+def count_total_tokens(messages):
+    total_tokens = 0
+    for message in messages:
+        total_tokens += count_tokens(message["content"])["n_tokens"]
+    return total_tokens

helper/question_templates.py

@@ -1136,12 +1136,11 @@ def getSpeakingTemplate():
         "exercises": [
             {
                 "id": str(uuid.uuid4()),
-                "prompts": [],
+                "prompts": ["questions"],
-                "text": "text",
+                "text": "Listen carefully and respond.",
-                "title": "topic",
+                "first_title": "first_topic",
-                "video_url": "sp1_video_url",
+                "second_title": "second_topic",
-                "video_path": "sp1_video_path",
+                "type": "interactiveSpeaking"
-                "type": "speaking"
             },
             {
                 "id": str(uuid.uuid4()),

helper/speech_to_text_helper.py

@@ -1,13 +1,15 @@
-import whisper
 import os
-import nltk
-import boto3
 import random
 
+import boto3
+import nltk
+import whisper
+
 nltk.download('words')
 from nltk.corpus import words
 from helper.constants import *
 
+
 def speech_to_text(file_path):
     if os.path.exists(file_path):
         model = whisper.load_model("base")
@@ -17,6 +19,7 @@ def speech_to_text(file_path):
         print("File not found:", file_path)
         raise Exception("File " + file_path + " not found.")
 
+
 def text_to_speech(text: str, file_name: str):
     # Initialize the Amazon Polly client
     client = boto3.client(
@@ -36,6 +39,15 @@ def text_to_speech(text: str, file_name: str):
             VoiceId=voice
         )
         audio_segments.append(tts_response['AudioStream'].read())
+
+    # Add finish message
+    audio_segments.append(client.synthesize_speech(
+        Engine="neural",
+        Text="This audio recording, for the listening exercise, has finished.",
+        OutputFormat="mp3",
+        VoiceId="Stephen"
+    )['AudioStream'].read())
+
     # Combine the audio segments into a single audio file
     combined_audio = b"".join(audio_segments)
     # Save the combined audio to a single file
@@ -44,6 +56,7 @@ def text_to_speech(text: str, file_name: str):
 
     print("Speech segments saved to " + file_name)
 
+
 def conversation_text_to_speech(conversation: list, file_name: str):
     # Initialize the Amazon Polly client
     client = boto3.client(
@@ -57,12 +70,21 @@ def conversation_text_to_speech(conversation: list, file_name: str):
     # Iterate through the text segments, convert to audio segments, and store them
     for segment in conversation:
         response = client.synthesize_speech(
-            Engine="neural", 
+            Engine="neural",
             Text=segment["text"],
             OutputFormat="mp3",
             VoiceId=segment["voice"]
         )
         audio_segments.append(response['AudioStream'].read())
+
+    # Add finish message
+    audio_segments.append(client.synthesize_speech(
+        Engine="neural",
+        Text="This audio recording, for the listening exercise, has finished.",
+        OutputFormat="mp3",
+        VoiceId="Stephen"
+    )['AudioStream'].read())
+
     # Combine the audio segments into a single audio file
     combined_audio = b"".join(audio_segments)
     # Save the combined audio to a single file
@@ -71,15 +93,27 @@ def conversation_text_to_speech(conversation: list, file_name: str):
 
     print("Speech segments saved to " + file_name)
 
+
 def has_words(text: str):
+    if not has_common_words(text):
+        return False
     english_words = set(words.words())
     words_in_input = text.split()
     return any(word.lower() in english_words for word in words_in_input)
 
-def has_10_words(text: str):
+
+def has_x_words(text: str, quantity):
+    if not has_common_words(text):
+        return False
     english_words = set(words.words())
     words_in_input = text.split()
     english_word_count = sum(1 for word in words_in_input if word.lower() in english_words)
+    return english_word_count >= quantity
+
+def has_common_words(text: str):
+    english_words = {"the", "be", "to", "of", "and", "a", "in", "that", "have", "i"}
+    words_in_input = text.split()
+    english_word_count = sum(1 for word in words_in_input if word.lower() in english_words)
     return english_word_count >= 10
 
 def divide_text(text, max_length=3000):
@@ -101,4 +135,4 @@ def divide_text(text, max_length=3000):
             divisions.append(text[current_position:next_position])
             current_position = next_position
 
-    return divisions
+    return divisions

helper/token_counter.py

@@ -1,5 +1,4 @@
 # This is a work in progress. There are still bugs. Once it is production-ready this will become a full repo.
-import os
 
 
 def count_tokens(text, model_name="gpt-3.5-turbo", debug=False):
@@ -86,4 +85,4 @@ class TokenBuffer:
             self.buffer = self.buffer.split(" ", removed_tokens)[-1]
 
     def get_buffer(self):
-        return self.buffer
+        return self.buffer

modules/__init__.py

@@ -0,0 +1,5 @@
+from .gpt import GPT
+
+__all__ = [
+    "GPT"
+]

modules/batch_users/__init__.py

@@ -0,0 +1,5 @@
+from .service import BatchUsers
+
+__all__ = [
+    "BatchUsers"
+]

modules/batch_users/batch_users.py

@@ -0,0 +1,31 @@
+import uuid
+from typing import Optional
+
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class DemographicInfo(BaseModel):
+    phone: str
+    passport_id: Optional[str] = None
+    country: Optional[str] = None
+
+
+class UserDTO(BaseModel):
+    id: uuid.UUID = Field(default_factory=uuid.uuid4)
+    email: str
+    name: str
+    type: str
+    passport_id: str
+    passwordHash: str
+    passwordSalt: str
+    groupName: Optional[str] = None
+    corporate: Optional[str] = None
+    studentID: Optional[str | int] = None
+    expiryDate: Optional[str] = None
+    demographicInformation: Optional[DemographicInfo] = None
+
+
+class BatchUsersDTO(BaseModel):
+    makerID: str
+    users: list[UserDTO]

modules/batch_users/service.py

@@ -0,0 +1,275 @@
+import os
+import subprocess
+import time
+import uuid
+from datetime import datetime
+from logging import getLogger
+
+import pandas as pd
+from typing import Dict
+
+import shortuuid
+from pymongo.database import Database
+
+from modules.batch_users.batch_users import BatchUsersDTO, UserDTO
+from modules.helper.file_helper import FileHelper
+
+
+class BatchUsers:
+
+    _DEFAULT_DESIRED_LEVELS = {
+        "reading": 9,
+        "listening": 9,
+        "writing": 9,
+        "speaking": 9,
+    }
+
+    _DEFAULT_LEVELS = {
+        "reading": 0,
+        "listening": 0,
+        "writing": 0,
+        "speaking": 0,
+    }
+
+    def __init__(self, mongo: Database):
+        self._db: Database = mongo
+        self._logger = getLogger(__name__)
+
+    def batch_users(self, request_data: Dict):
+        batch_dto = self._map_to_batch(request_data)
+
+        file_name = f'{uuid.uuid4()}.csv'
+        path = f'./tmp/{file_name}'
+        self._generate_firebase_auth_csv(batch_dto, path)
+
+        result = self._upload_users('./tmp', file_name)
+        if result.returncode != 0:
+            error_msg = f"Couldn't upload users. Failed to run command firebase auth import -> ```cmd {result.stdout}```"
+            self._logger.error(error_msg)
+            return error_msg
+
+        self._init_users(batch_dto)
+
+        FileHelper.remove_file(path)
+        return {"ok": True}
+
+    @staticmethod
+    def _map_to_batch(request_data: Dict) -> BatchUsersDTO:
+        users_list = [{**user} for user in request_data["users"]]
+        for user in users_list:
+            user["studentID"] = str(user["studentID"])
+
+        users: list[UserDTO] = [UserDTO(**user) for user in users_list]
+        return BatchUsersDTO(makerID=request_data["makerID"], users=users)
+
+    @staticmethod
+    def _generate_firebase_auth_csv(batch_dto: BatchUsersDTO, path: str):
+        # https://firebase.google.com/docs/cli/auth#file_format
+        columns = [
+            'UID', 'Email', 'Email Verified', 'Password Hash', 'Password Salt', 'Name',
+            'Photo URL', 'Google ID', 'Google Email', 'Google Display Name', 'Google Photo URL',
+            'Facebook ID', 'Facebook Email', 'Facebook Display Name', 'Facebook Photo URL',
+            'Twitter ID', 'Twitter Email', 'Twitter Display Name', 'Twitter Photo URL',
+            'GitHub ID', 'GitHub Email', 'GitHub Display Name', 'GitHub Photo URL',
+            'User Creation Time', 'Last Sign-In Time', 'Phone Number'
+        ]
+        users_data = []
+
+        current_time = int(time.time() * 1000)
+
+        for user in batch_dto.users:
+            user_data = {
+                'UID': str(user.id),
+                'Email': user.email,
+                'Email Verified': False,
+                'Password Hash': user.passwordHash,
+                'Password Salt': user.passwordSalt,
+                'Name': '',
+                'Photo URL': '',
+                'Google ID': '',
+                'Google Email': '',
+                'Google Display Name': '',
+                'Google Photo URL': '',
+                'Facebook ID': '',
+                'Facebook Email': '',
+                'Facebook Display Name': '',
+                'Facebook Photo URL': '',
+                'Twitter ID': '',
+                'Twitter Email': '',
+                'Twitter Display Name': '',
+                'Twitter Photo URL': '',
+                'GitHub ID': '',
+                'GitHub Email': '',
+                'GitHub Display Name': '',
+                'GitHub Photo URL': '',
+                'User Creation Time': current_time,
+                'Last Sign-In Time': '',
+                'Phone Number': ''
+            }
+            users_data.append(user_data)
+
+        df = pd.DataFrame(users_data, columns=columns)
+        df.to_csv(path, index=False, header=False)
+
+    @staticmethod
+    def _upload_users(directory: str, file_name: str):
+        command = (
+            f'firebase auth:import {file_name} '
+            f'--hash-algo=SCRYPT '
+            f'--hash-key={os.getenv("FIREBASE_SCRYPT_B64_SIGNER_KEY")} '
+            f'--salt-separator={os.getenv("FIREBASE_SCRYPT_B64_SALT_SEPARATOR")} '
+            f'--rounds={os.getenv("FIREBASE_SCRYPT_ROUNDS")} '
+            f'--mem-cost={os.getenv("FIREBASE_SCRYPT_MEM_COST")} '
+            f'--project={os.getenv("FIREBASE_PROJECT_ID")} '
+        )
+
+        result = subprocess.run(command, shell=True, cwd=directory, capture_output=True, text=True)
+        return result
+
+    def _init_users(self, batch_users: BatchUsersDTO):
+        maker_id = batch_users.makerID
+        for user in batch_users.users:
+            self._insert_new_user(user)
+            code = self._create_code(user, maker_id)
+
+            if user.type == "corporate":
+                self._set_corporate_default_groups(user)
+
+            if user.corporate:
+                self._assign_corporate_to_user(user, code)
+
+            if user.groupName and len(user.groupName.strip()) > 0:
+                self._assign_user_to_group_by_name(user, maker_id)
+
+    def _insert_new_user(self, user: UserDTO):
+        new_user = {
+            **user.dict(exclude={
+                'passport_id', 'groupName', 'expiryDate',
+                'corporate', 'passwordHash', 'passwordSalt'
+            }),
+            'id': str(user.id),
+            'bio': "",
+            'focus': "academic",
+            'status': "active",
+            'desiredLevels': self._DEFAULT_DESIRED_LEVELS,
+            'profilePicture': "/defaultAvatar.png",
+            'levels': self._DEFAULT_LEVELS,
+            'isFirstLogin': False,
+            'isVerified': True,
+            'registrationDate': datetime.now(),
+            'subscriptionExpirationDate': user.expiryDate
+        }
+        self._db.users.insert_one(new_user)
+
+    def _create_code(self, user: UserDTO, maker_id: str) -> str:
+        code = shortuuid.ShortUUID().random(length=6)
+        self._db.codes.insert_one({
+            'id': code,
+            'code': code,
+            'creator': maker_id,
+            'expiryDate': user.expiryDate,
+            'type': user.type,
+            'creationDate': datetime.now(),
+            'userId': str(user.id),
+            'email': user.email,
+            'name': user.name,
+            'passport_id': user.passport_id
+        })
+        return code
+
+    def _set_corporate_default_groups(self, user: UserDTO):
+        user_id = str(user.id)
+        default_groups = [
+            {
+                'admin': user_id,
+                'id': str(uuid.uuid4()),
+                'name': "Teachers",
+                'participants': [],
+                'disableEditing': True,
+            },
+            {
+                'admin': user_id,
+                'id': str(uuid.uuid4()),
+                'name': "Students",
+                'participants': [],
+                'disableEditing': True,
+            },
+            {
+                'admin': user_id,
+                'id': str(uuid.uuid4()),
+                'name': "Corporate",
+                'participants': [],
+                'disableEditing': True,
+            }
+        ]
+        for group in default_groups:
+            self._db.groups.insert_one(group)
+
+    def _assign_corporate_to_user(self, user: UserDTO, code: str):
+        user_id = str(user.id)
+        corporate_user = self._db.users.find_one(
+            {"email": user.corporate}
+        )
+        if corporate_user:
+            self._db.codes.update_one(
+                {"id": code},
+                {"$set": {"creator": corporate_user["id"]}},
+                upsert=True
+            )
+            group_type = "Students" if user.type == "student" else "Teachers"
+
+            group = self._db.groups.find_one(
+                {
+                    "admin": corporate_user["id"],
+                    "name": group_type
+                }
+            )
+
+            if group:
+                participants = group['participants']
+                if user_id not in participants:
+                    participants.append(user_id)
+                    self._db.groups.update_one(
+                        {"id": group["id"]},
+                        {"$set": {"participants": participants}}
+                    )
+                    
+            else:
+                group = {
+                    'admin': corporate_user["id"],
+                    'id': str(uuid.uuid4()),
+                    'name': group_type,
+                    'participants': [user_id],
+                    'disableEditing': True,
+                }
+                
+                self._db.groups.insert_one(group)
+
+    def _assign_user_to_group_by_name(self, user: UserDTO, maker_id: str):
+        user_id = str(user.id)
+
+        groups = list(self._db.groups.find(
+            {
+                "admin": maker_id,
+                "name": user.groupName.strip()
+            }
+        ))
+
+        if len(groups) == 0:
+            new_group = {
+                'id': str(uuid.uuid4()),
+                'admin': maker_id,
+                'name': user.groupName.strip(),
+                'participants': [user_id],
+                'disableEditing': False,
+            }
+            self._db.groups.insert_one(new_group)
+        else:
+            group = groups[0]
+            participants = group["participants"]
+            if user_id not in participants:
+                participants.append(user_id)
+                self._db.groups.update_one(
+                    {"id": group["id"]},
+                    {"$set": {"participants": participants}}
+                )

modules/gpt.py

@@ -0,0 +1,66 @@
+import json
+from logging import getLogger
+
+from typing import List, Optional, Callable, TypeVar
+
+from openai.types.chat import ChatCompletionMessageParam
+from pydantic import BaseModel
+
+T = TypeVar('T', bound=BaseModel)
+
+
+class GPT:
+
+    def __init__(self, openai_client):
+        self._client = openai_client
+        self._default_model = "gpt-4o-2024-08-06"
+        self._logger = getLogger(__name__)
+
+    def prediction(
+            self,
+            messages: List[ChatCompletionMessageParam],
+            map_to_model: Callable,
+            json_scheme: str,
+            *,
+            model: Optional[str] = None,
+            temperature: Optional[float] = None,
+            max_retries: int = 3
+    ) -> List[T] | T | None:
+        params = {
+            "messages": messages,
+            "response_format": {"type": "json_object"},
+            "model": model if model else self._default_model
+        }
+
+        if temperature:
+            params["temperature"] = temperature
+
+        attempt = 0
+        while attempt < max_retries:
+            result = self._client.chat.completions.create(**params)
+            result_content = result.choices[0].message.content
+            try:
+                result_json = json.loads(result_content)
+                return map_to_model(result_json)
+            except Exception as e:
+                attempt += 1
+                self._logger.info(f"GPT returned malformed response: {result_content}\n {str(e)}")
+                params["messages"] = [
+                    {
+                        "role": "user",
+                        "content": (
+                            "Your previous response wasn't in the json format I've explicitly told you to output. "
+                            f"In your next response, you will fix it and return me just the json I've asked."
+                        )
+                    },
+                    {
+                        "role": "user",
+                        "content": (
+                            f"Previous response: {result_content}\n"
+                            f"JSON format: {json_scheme}"
+                        )
+                    }
+                ]
+                if attempt >= max_retries:
+                    self._logger.error(f"Max retries exceeded!")
+                    return None

modules/helper/__init__.py

@@ -0,0 +1,5 @@
+from .logger import LoggerHelper
+
+__all__ = [
+    "LoggerHelper"
+]

modules/helper/file_helper.py

@@ -0,0 +1,97 @@
+import base64
+import io
+import os
+import shutil
+import subprocess
+import uuid
+from typing import Optional, Tuple
+
+import numpy as np
+import pypandoc
+from PIL import Image
+
+
+class FileHelper:
+
+    # Supposedly pandoc covers a wide range of file extensions only tested with docx
+    @staticmethod
+    def convert_file_to_pdf(input_path: str, output_path: str):
+        pypandoc.convert_file(input_path, 'pdf', outputfile=output_path, extra_args=[
+            '-V', 'geometry:paperwidth=5.5in',
+            '-V', 'geometry:paperheight=8.5in',
+            '-V', 'geometry:margin=0.5in',
+            '-V', 'pagestyle=empty'
+        ])
+
+    @staticmethod
+    def convert_file_to_html(input_path: str, output_path: str):
+        pypandoc.convert_file(input_path, 'html', outputfile=output_path)
+
+    @staticmethod
+    def pdf_to_png(path_id: str):
+        to_png = f"pdftoppm -png exercises.pdf page"
+        result = subprocess.run(to_png, shell=True, cwd=f'./tmp/{path_id}', capture_output=True, text=True)
+        if result.returncode != 0:
+            raise Exception(
+                f"Couldn't convert pdf to png. Failed to run command '{to_png}' -> ```cmd {result.stderr}```")
+
+    @staticmethod
+    def is_page_blank(image_bytes: bytes, image_threshold=10) -> bool:
+        with Image.open(io.BytesIO(image_bytes)) as img:
+            img_gray = img.convert('L')
+            img_array = np.array(img_gray)
+            non_white_pixels = np.sum(img_array < 255)
+
+        return non_white_pixels <= image_threshold
+
+    @classmethod
+    def _encode_image(cls, image_path: str, image_threshold=10) -> Optional[str]:
+        with open(image_path, "rb") as image_file:
+            image_bytes = image_file.read()
+
+        if cls.is_page_blank(image_bytes, image_threshold):
+            return None
+
+        return base64.b64encode(image_bytes).decode('utf-8')
+
+    @classmethod
+    def b64_pngs(cls, path_id: str, files: list[str]):
+        png_messages = []
+        for filename in files:
+            b64_string = cls._encode_image(os.path.join(f'./tmp/{path_id}', filename))
+            if b64_string:
+                png_messages.append({
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/png;base64,{b64_string}"
+                    }
+                })
+        return png_messages
+
+    @staticmethod
+    def remove_directory(path):
+        try:
+            if os.path.exists(path):
+                if os.path.isdir(path):
+                    shutil.rmtree(path)
+        except Exception as e:
+            print(f"An error occurred while trying to remove {path}: {str(e)}")
+
+    @staticmethod
+    def remove_file(file_path):
+        try:
+            if os.path.exists(file_path):
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+        except Exception as e:
+            print(f"An error occurred while trying to remove the file {file_path}: {str(e)}")
+
+    @staticmethod
+    def save_upload(file) -> Tuple[str, str]:
+        ext = file.filename.split('.')[-1]
+        path_id = str(uuid.uuid4())
+        os.makedirs(f'./tmp/{path_id}', exist_ok=True)
+
+        tmp_filename = f'./tmp/{path_id}/uploaded.{ext}'
+        file.save(tmp_filename)
+        return ext, path_id

modules/helper/logger.py

@@ -0,0 +1,23 @@
+import logging
+from functools import wraps
+
+
+class LoggerHelper:
+
+    @staticmethod
+    def suppress_loggers():
+        def decorator(f):
+            @wraps(f)
+            def wrapped(*args, **kwargs):
+                root_logger = logging.getLogger()
+                original_level = root_logger.level
+
+                root_logger.setLevel(logging.ERROR)
+
+                try:
+                    return f(*args, **kwargs)
+                finally:
+                    root_logger.setLevel(original_level)
+
+            return wrapped
+        return decorator

modules/training_content/__init__.py

@@ -0,0 +1,7 @@
+from .kb import TrainingContentKnowledgeBase
+from .service import TrainingContentService
+
+__all__ = [
+    "TrainingContentService",
+    "TrainingContentKnowledgeBase"
+]

modules/training_content/dtos.py

@@ -0,0 +1,29 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class QueryDTO(BaseModel):
+    category: str
+    text: str
+
+
+class DetailsDTO(BaseModel):
+    exam_id: str
+    date: int
+    performance_comment: str
+    detailed_summary: str
+
+
+class WeakAreaDTO(BaseModel):
+    area: str
+    comment: str
+
+
+class TrainingContentDTO(BaseModel):
+    details: List[DetailsDTO]
+    weak_areas: List[WeakAreaDTO]
+    queries: List[QueryDTO]
+
+
+class TipsDTO(BaseModel):
+    tip_ids: List[str]

modules/training_content/kb.py

@@ -0,0 +1,85 @@
+import json
+import os
+from logging import getLogger
+from typing import Dict, List
+
+import faiss
+import pickle
+
+
+class TrainingContentKnowledgeBase:
+
+    def __init__(self, embeddings, path: str = 'pathways_2_rw_with_ids.json'):
+        self._embedding_model = embeddings
+        self._tips = None  # self._read_json(path)
+        self._category_metadata = None
+        self._indices = None
+        self._logger = getLogger(__name__)
+
+    @staticmethod
+    def _read_json(path: str) -> Dict[str, any]:
+        with open(path, 'r', encoding="utf-8") as json_file:
+            return json.loads(json_file.read())
+
+    def print_category_count(self):
+        category_tips = {}
+        for unit in self._tips['units']:
+            for page in unit['pages']:
+                for tip in page['tips']:
+                    category = tip['category'].lower().replace(" ", "_")
+                    if category not in category_tips:
+                        category_tips[category] = 0
+                    else:
+                        category_tips[category] = category_tips[category] + 1
+        print(category_tips)
+
+    def create_embeddings_and_save_them(self) -> None:
+        category_embeddings = {}
+        category_metadata = {}
+
+        for unit in self._tips['units']:
+            for page in unit['pages']:
+                for tip in page['tips']:
+                    category = tip['category'].lower().replace(" ", "_")
+                    if category not in category_embeddings:
+                        category_embeddings[category] = []
+                        category_metadata[category] = []
+
+                    category_embeddings[category].append(tip['embedding'])
+                    category_metadata[category].append({"id": tip['id'], "text": tip['text']})
+
+        category_indices = {}
+        for category, embeddings in category_embeddings.items():
+            embeddings_array = self._embedding_model.encode(embeddings)
+            index = faiss.IndexFlatL2(embeddings_array.shape[1])
+            index.add(embeddings_array)
+            category_indices[category] = index
+
+            faiss.write_index(index, f"./faiss/{category}_tips_index.faiss")
+
+        with open("./faiss/tips_metadata.pkl", "wb") as f:
+            pickle.dump(category_metadata, f)
+
+    def load_indices_and_metadata(
+            self,
+            directory: str = './faiss',
+            suffix: str = '_tips_index.faiss',
+            metadata_path: str = './faiss/tips_metadata.pkl'
+    ):
+        files = os.listdir(directory)
+        self._indices = {}
+        for file in files:
+            if file.endswith(suffix):
+                self._indices[file[:-len(suffix)]] = faiss.read_index(f'{directory}/{file}')
+                self._logger.info(f'Loaded embeddings for {file[:-len(suffix)]} category.')
+
+        with open(metadata_path, 'rb') as f:
+            self._category_metadata = pickle.load(f)
+        self._logger.info("Loaded tips metadata")
+
+    def query_knowledge_base(self, query: str, category: str, top_k: int = 5) -> List[Dict[str, str]]:
+        query_embedding = self._embedding_model.encode([query])
+        index = self._indices[category]
+        D, I = index.search(query_embedding, top_k)
+        results = [self._category_metadata[category][i] for i in I[0]]
+        return results

modules/training_content/service.py

@@ -0,0 +1,407 @@
+import json
+import uuid
+from datetime import datetime
+from logging import getLogger
+
+from typing import Dict, List
+
+from pymongo.database import Database
+
+from modules.training_content.dtos import TrainingContentDTO, WeakAreaDTO, QueryDTO, DetailsDTO, TipsDTO
+
+
+class TrainingContentService:
+
+    TOOLS = [
+        'critical_thinking',
+        'language_for_writing',
+        'reading_skills',
+        'strategy',
+        'words',
+        'writing_skills'
+    ]
+    # strategy word_link ct_focus reading_skill word_partners writing_skill language_for_writing
+
+    def __init__(self, kb, openai, mongo: Database):
+        self._training_content_module = kb
+        self._db: Database = mongo
+        self._logger = getLogger(__name__)
+        self._llm = openai
+
+    def get_tips(self, training_content):
+        user, stats = training_content["userID"], training_content["stats"]
+        exam_data, exam_map = self._sort_out_solutions(stats)
+        training_content = self._get_exam_details_and_tips(exam_data)
+        tips = self._query_kb(training_content.queries)
+        usefull_tips = self._get_usefull_tips(exam_data, tips)
+        exam_map = self._merge_exam_map_with_details(exam_map, training_content.details)
+
+        weak_areas = {"weak_areas": []}
+        for area in training_content.weak_areas:
+            weak_areas["weak_areas"].append(area.dict())
+
+        new_id = str(uuid.uuid4())
+        training_doc = {
+            'id': new_id,
+            'created_at': int(datetime.now().timestamp() * 1000),
+            **exam_map,
+            **usefull_tips.dict(),
+            **weak_areas,
+            "user": user
+        }
+        self._db.training.insert_one(training_doc)
+        return {
+            "id": new_id
+        }
+
+    @staticmethod
+    def _merge_exam_map_with_details(exam_map: Dict[str, any], details: List[DetailsDTO]):
+        new_exam_map = {"exams": []}
+        for detail in details:
+            new_exam_map["exams"].append({
+                "id": detail.exam_id,
+                "date": detail.date,
+                "performance_comment": detail.performance_comment,
+                "detailed_summary": detail.detailed_summary,
+                **exam_map[detail.exam_id]
+            })
+        return new_exam_map
+
+    def _query_kb(self, queries: List[QueryDTO]):
+        map_categories = {
+            "critical_thinking": "ct_focus",
+            "language_for_writing": "language_for_writing",
+            "reading_skills": "reading_skill",
+            "strategy": "strategy",
+            "writing_skills": "writing_skill"
+        }
+
+        tips = {"tips": []}
+        for query in queries:
+            if query.category == "words":
+                tips["tips"].extend(
+                    self._training_content_module.query_knowledge_base(query.text, "word_link")
+                )
+                tips["tips"].extend(
+                    self._training_content_module.query_knowledge_base(query.text, "word_partners")
+                )
+            else:
+                if query.category in map_categories:
+                    tips["tips"].extend(
+                        self._training_content_module.query_knowledge_base(query.text, map_categories[query.category])
+                    )
+                else:
+                    self._logger.info(f"GTP tried to query knowledge base for {query.category} and it doesn't exist.")
+        return tips
+
+    def _get_exam_details_and_tips(self, exam_data: Dict[str, any]) -> TrainingContentDTO:
+        json_schema = (
+            '{ "details": [{"exam_id": "", "date": 0, "performance_comment": "", "detailed_summary": ""}],'
+            ' "weak_areas": [{"area": "", "comment": ""}], "queries": [{"text": "", "category": ""}] }'
+        )
+        messages = [
+            {
+                "role": "user",
+                "content": (
+                    f"I'm going to provide you with exam data, you will take the exam data and fill this json "
+                    f'schema : {json_schema}. "performance_comment" is a short sentence that describes the '
+                    'students\'s performance and main mistakes in a single exam, "detailed_summary" is a detailed '
+                    'summary of the student\'s performance, "weak_areas" are identified areas'
+                    ' across all exams which need to be improved upon, for example, area "Grammar and Syntax" comment "Issues'
+                    ' with sentence structure and punctuation.", the "queries" field is where you will write queries '
+                    'for tips that will be displayed to the student, the category attribute is a collection of '
+                    'embeddings and the text will be the text used to query the knowledge base. The categories are '
+                    f'the following [{", ".join(self.TOOLS)}]. The exam data will be a json where the key of the field '
+                    '"exams" is the exam id, an exam can be composed of multiple modules or single modules. The student'
+                    ' will see your response so refrain from using phrasing like "The student" did x, y and z. If the '
+                    'field "answer" in a question is an empty array "[]", then the student didn\'t answer any question '
+                    'and you must address that in your response. Also questions aren\'t modules, the only modules are: '
+                    'level, speaking, writing, reading and listening. The details array needs to be tailored to the '
+                    'exam attempt, even if you receive the same exam you must treat as different exams by their id.'
+                    'Don\'t make references to an exam by it\'s id, the GUI will handle that so the student knows '
+                    'which is the exam your comments and summary are referencing too. Even if the student hasn\'t '
+                    'submitted no answers for an exam, you must still fill the details structure addressing that fact.'
+                )
+            },
+            {
+                "role": "user",
+                "content": f'Exam Data: {str(exam_data)}'
+            }
+        ]
+        return self._llm.prediction(messages, self._map_gpt_response, json_schema)
+
+    def _get_usefull_tips(self, exam_data: Dict[str, any], tips: Dict[str, any]) -> TipsDTO:
+        json_schema = (
+            '{ "tip_ids": [] }'
+        )
+        messages = [
+            {
+                "role": "user",
+                "content": (
+                    f"I'm going to provide you with tips and I want you to return to me the tips that "
+                    f"can be usefull for the student that made the exam that I'm going to send you, return "
+                    f"me the tip ids in this json format {json_schema}."
+                )
+            },
+            {
+                "role": "user",
+                "content": f'Exam Data: {str(exam_data)}'
+            },
+            {
+                "role": "user",
+                "content": f'Tips: {str(tips)}'
+            }
+        ]
+        return self._llm.prediction(messages, lambda response: TipsDTO(**response), json_schema)
+
+    @staticmethod
+    def _map_gpt_response(response: Dict[str, any]) -> TrainingContentDTO:
+        parsed_response = {
+            "details": [DetailsDTO(**detail) for detail in response["details"]],
+            "weak_areas": [WeakAreaDTO(**area) for area in response["weak_areas"]],
+            "queries": [QueryDTO(**query) for query in response["queries"]]
+        }
+        return TrainingContentDTO(**parsed_response)
+
+    def _sort_out_solutions(self, stats):
+        grouped_stats = {}
+        for stat in stats:
+            session_key = f'{str(stat["date"])}-{stat["user"]}'
+            module = stat["module"]
+            exam_id = stat["exam"]
+
+            if session_key not in grouped_stats:
+                grouped_stats[session_key] = {}
+            if module not in grouped_stats[session_key]:
+                grouped_stats[session_key][module] = {
+                    "stats": [],
+                    "exam_id": exam_id
+                }
+            grouped_stats[session_key][module]["stats"].append(stat)
+
+        exercises = {}
+        exam_map = {}
+        for session_key, modules in grouped_stats.items():
+            exercises[session_key] = {}
+            for module, module_stats in modules.items():
+                exercises[session_key][module] = {}
+
+                exam_id = module_stats["exam_id"]
+                if exam_id not in exercises[session_key][module]:
+                    exercises[session_key][module][exam_id] = {"date": None, "exercises": []}
+
+                exam_total_questions = 0
+                exam_total_correct = 0
+
+                for stat in module_stats["stats"]:
+                    exam_total_questions += stat["score"]["total"]
+                    exam_total_correct += stat["score"]["correct"]
+                    exercises[session_key][module][exam_id]["date"] = stat["date"]
+
+                    if session_key not in exam_map:
+                        exam_map[session_key] = {"stat_ids": [], "score": 0}
+                    exam_map[session_key]["stat_ids"].append(stat["id"])
+
+                    exam = self._get_doc_by_id(module, exam_id)
+                    if module == "listening":
+                        exercises[session_key][module][exam_id]["exercises"].extend(
+                            self._get_listening_solutions(stat, exam))
+                    elif module == "reading":
+                        exercises[session_key][module][exam_id]["exercises"].extend(
+                            self._get_reading_solutions(stat, exam))
+                    elif module == "writing":
+                        exercises[session_key][module][exam_id]["exercises"].extend(
+                            self._get_writing_prompts_and_answers(stat, exam)
+                        )
+                    elif module == "speaking":
+                        exercises[session_key][module][exam_id]["exercises"].extend(
+                            self._get_speaking_solutions(stat, exam)
+                        )
+                    elif module == "level":
+                        exercises[session_key][module][exam_id]["exercises"].extend(
+                            self._get_level_solutions(stat, exam)
+                        )
+
+                exam_map[session_key]["score"] = round((exam_total_correct / exam_total_questions) * 100)
+                exam_map[session_key]["module"] = module
+
+        return {"exams": exercises}, exam_map
+
+    def _get_writing_prompts_and_answers(self, stat, exam):
+        result = []
+        try:
+            exercises = []
+            for solution in stat['solutions']:
+                answer = solution['solution']
+                exercise_id = solution['id']
+                exercises.append({
+                    "exercise_id": exercise_id,
+                    "answer": answer
+                })
+            for exercise in exercises:
+                for exam_exercise in exam["exercises"]:
+                    if exam_exercise["id"] == exercise["exercise_id"]:
+                        result.append({
+                            "exercise": exam_exercise["prompt"],
+                            "answer": exercise["answer"]
+                        })
+
+        except KeyError as e:
+            self._logger.warning(f"Malformed stat object: {str(e)}")
+
+        return result
+
+    @staticmethod
+    def _get_mc_question(exercise, stat):
+        shuffle_maps = stat.get("shuffleMaps", [])
+        answer = stat["solutions"] if len(shuffle_maps) == 0 else []
+        if len(shuffle_maps) != 0:
+            for solution in stat["solutions"]:
+                shuffle_map = [
+                    item["map"] for item in shuffle_maps
+                    if item["questionID"] == solution["question"]
+                ]
+                answer.append({
+                    "question": solution["question"],
+                    "option": shuffle_map[solution["option"]]
+                })
+        return {
+            "question": exercise["prompt"],
+            "exercise": exercise["questions"],
+            "answer": stat["solutions"]
+        }
+
+    @staticmethod
+    def _swap_key_name(d, original_key, new_key):
+        d[new_key] = d.pop(original_key)
+        return d
+
+    def _get_level_solutions(self, stat, exam):
+        result = []
+        try:
+            for part in exam["parts"]:
+                for exercise in part["exercises"]:
+                    if exercise["id"] == stat["exercise"]:
+                        if stat["type"] == "fillBlanks":
+                            result.append({
+                                "prompt": exercise["prompt"],
+                                "template": exercise["text"],
+                                "words": exercise["words"],
+                                "solutions": exercise["solutions"],
+                                "answer": [
+                                    self._swap_key_name(item, 'solution', 'option')
+                                    for item in stat["solutions"]
+                                ]
+                            })
+                        elif stat["type"] == "multipleChoice":
+                            result.append(self._get_mc_question(exercise, stat))
+        except KeyError as e:
+            self._logger.warning(f"Malformed stat object: {str(e)}")
+        return result
+
+    def _get_listening_solutions(self, stat, exam):
+        result = []
+        try:
+            for part in exam["parts"]:
+                for exercise in part["exercises"]:
+                    if exercise["id"] == stat["exercise"]:
+                        if stat["type"] == "writeBlanks":
+                            result.append({
+                                "question": exercise["prompt"],
+                                "template": exercise["text"],
+                                "solution": exercise["solutions"],
+                                "answer": stat["solutions"]
+                            })
+                        elif stat["type"] == "fillBlanks":
+                            result.append({
+                                "question": exercise["prompt"],
+                                "template": exercise["text"],
+                                "words": exercise["words"],
+                                "solutions": exercise["solutions"],
+                                "answer": stat["solutions"]
+                            })
+                        elif stat["type"] == "multipleChoice":
+                            result.append(self._get_mc_question(exercise, stat))
+
+        except KeyError as e:
+            self._logger.warning(f"Malformed stat object: {str(e)}")
+        return result
+
+    @staticmethod
+    def _find_shuffle_map(shuffle_maps, question_id):
+        return next((item["map"] for item in shuffle_maps if item["questionID"] == question_id), None)
+
+    def _get_speaking_solutions(self, stat, exam):
+        result = {}
+        try:
+            result = {
+                "comments": {
+                    key: value['comment'] for key, value in stat['solutions'][0]['evaluation']['task_response'].items()}
+                ,
+                "exercises": {}
+            }
+
+            for exercise in exam["exercises"]:
+                if exercise["id"] == stat["exercise"]:
+                    if stat["type"] == "interactiveSpeaking":
+                        for i in range(len(exercise["prompts"])):
+                            result["exercises"][f"exercise_{i+1}"] = {
+                                "question": exercise["prompts"][i]["text"]
+                            }
+                        for i in range(len(exercise["prompts"])):
+                            answer = stat['solutions'][0]["evaluation"].get(f'transcript_{i+1}', '')
+                            result["exercises"][f"exercise_{i+1}"]["answer"] = answer
+                    elif stat["type"] == "speaking":
+                        result["exercises"]["exercise_1"] = {
+                            "question": exercise["text"],
+                            "answer": stat['solutions'][0]["evaluation"].get(f'transcript', '')
+                        }
+        except KeyError as e:
+            self._logger.warning(f"Malformed stat object: {str(e)}")
+        return [result]
+
+    def _get_reading_solutions(self, stat, exam):
+        result = []
+        try:
+            for part in exam["parts"]:
+                text = part["text"]
+                for exercise in part["exercises"]:
+                    if exercise["id"] == stat["exercise"]:
+                        if stat["type"] == "fillBlanks":
+                            result.append({
+                                "text": text,
+                                "question": exercise["prompt"],
+                                "template": exercise["text"],
+                                "words": exercise["words"],
+                                "solutions": exercise["solutions"],
+                                "answer": stat["solutions"]
+                            })
+                        elif stat["type"] == "writeBlanks":
+                            result.append({
+                                "text": text,
+                                "question": exercise["prompt"],
+                                "template": exercise["text"],
+                                "solutions": exercise["solutions"],
+                                "answer": stat["solutions"]
+                            })
+                        elif stat["type"] == "trueFalse":
+                            result.append({
+                                "text": text,
+                                "questions": exercise["questions"],
+                                "answer": stat["solutions"]
+                            })
+                        elif stat["type"] == "matchSentences":
+                            result.append({
+                                "text": text,
+                                "question": exercise["prompt"],
+                                "sentences": exercise["sentences"],
+                                "options": exercise["options"],
+                                "answer": stat["solutions"]
+                            })
+        except KeyError as e:
+            self._logger.warning(f"Malformed stat object: {str(e)}")
+        return result
+
+    def _get_doc_by_id(self, collection: str, doc_id: str):
+        doc = self._db[collection].find_one({"id": doc_id})
+        return doc

modules/training_content/tips/instructions.MD

@@ -0,0 +1,67 @@
+# Adding new training content
+
+If you're ever tasked with the grueling task of adding more tips from manuals, my condolences.
+
+There are 4 components of a training content tip: the tip itself, the question, the additional and the segment.
+
+The tip is the actual tip, if the manual doesn't have an exercise that relates to that tip fill this out:
+
+```json
+{
+    "category": "<the category of the tip that will be used to categorize the embeddings and also used in the tip header>",
+    "embedding": "<the relevant part of the tip that is needed to make the embedding (clean the tip of useless info that might mislead the queries)>",
+    "text": "<The text that the llm will use to assess whether the tip is relevant according to the performance of the student (most of the time just include all the text of the tip)>",
+    "html": "<The html that will be rendered in the tip component>",
+    "id": "<a uuid4>",
+    "verified": <this is just to keep track of the tips that were manually confirmed by you>,
+    "standalone": <if the tip doesn't have an exercise this is true else it's false>
+}
+```
+
+If the manual does have an exercise that relates to the tip:
+
+```json
+{
+    // ...    
+    "question": "<the exercise question(s) html>",
+    "additional": "<context of the question html>",
+    "segments": [
+        {
+            "html": "<the html of a segment, you MUST wrap the html in a single <div> >",
+            "wordDelay": <the speed at which letters will be placed on the segment, 200ms is a good one>,
+            "holdDelay": <the total time that the segment will be paused before moving onto the next segment, 5000ms is a good one>,
+            "highlight": [
+                {
+                    "targets": ["<the target of the highlight can be: question, additional, segment, all>"],
+                    "phrases": ["<the words/phrases/raw html you want to highlight>"]
+                }
+            ],
+            "insertHTML": [
+                {
+                    "target": "<the target of the insert can be: question, additional>",
+                    "targetId": "<the id of an html element>",
+                    "position": "<the position of the inserted html can be: replace, prepend and append. Most of the time you will only use replace>",
+                    "html": "<the html to replace the element with targetId>"
+                },
+            ]
+        }
+    ]
+}
+```
+
+In order to create these structures you will have to mannually screenshot the tips, exercises, context and send them to an llm (gpt-4o or claude)
+with a prompt like "get me the html for this", you will have to check whether the html is properly structured and then
+paste them in the prompt.txt file of this directory and send it 
+back to an llm.
+
+Afterwards you will have to check whether the default styles in /src/components/TrainingContent/FormatTip.ts are adequate, divs 
+(except for the wrapper div of a segment) and span styles are not overriden but you should aim to use the least ammount of
+styles in the tip itself and create custom reusable html elements
+in FormatTip.ts.
+
+After checking all of the tips render you will have to create new embeddings in the backend, you CAN'T change ids of existing tips since there 
+might be training tips that are already stored in firebase.
+
+This is a very tedious task here's a recommendation for [background noise](https://www.youtube.com/watch?v=lDnva_3fcTc).
+
+GL HF 

modules/training_content/tips/prompt.txt

@@ -0,0 +1,62 @@
+I am going to give you an exercise and a tip, explain how to solve the exercise and how the tip is beneficial, 
+your response must be with this format:
+
+{
+  "segments": [
+    {
+        "html": "",
+        "wordDelay": 0,
+        "holdDelay"; 0,
+        "highlight": [
+          {
+              "targets": [],
+              "phrases": []
+          }
+        ],
+        "insertHTML": [
+          {
+              "target": "",
+              "targetId": "",
+              "position": "replace",
+              "html": ""
+          }
+        ]
+    }
+  ]
+}
+
+Basically you are going to produce multiple objects and place it in data with the format above to integrate with a react component that highlights passages and inserts html, 
+these objects are segments of your explanation that will be presented to a student. 
+
+In the html field place a segment of your response that will be streamed to the component with a delay of "wordDelay" ms and in the end of that segment stream the phrases or words inside 
+"highlight" will be highlighted for "holdDelay" ms, and the cycle repeats until the whole data array is iterated. Make it so
+that the delays are reasonable for the student have time to process the message your trying to send. Take note that 
+"wordDelay" is the time between words to display (always 200), and "holdDelay" (no less than 5000) is the total time the highlighter will highlight what you put
+inside "highlight". 
+
+There are 3 target areas:
+- "question": where the question is placed
+- "additional": where additional content is placed required to answer the question (this section is optional)
+- "segment": a particular segment
+
+You can use these targets in highlight and insertHTML. In order for insertHTML to work, you will have to place an html element with an "id" attribute
+in the targets you will reference and provide the id via the "targetId", by this I mean if you want to use insert you will need to provide me the 
+html I've sent you with either a placeholder element with an id set or set an id in an existent element.
+
+If there are already id's in the html I'm giving you then you must use insertHtml.
+
+Each segment html will be rendered in a div that as margins, you should condense the information don't give me just single short phrases that occupy a whole div.
+As previously said this wil be seen by a student so show some train of thought to solve the exercise.
+All the segment's html must be wrapped in a div element, and again since this div element will be rendered with some margins make proper use of the segments html.
+
+Try to make bulletpoints.
+Dont explicitely mention the tip right away at the beginning, aim more towards the end.
+
+
+Tip:
+
+
+Target: "question"
+
+
+Target: "additional"

modules/training_content/tips/send_tips_to_firestore.py

@@ -0,0 +1,34 @@
+import json
+import os
+
+from dotenv import load_dotenv
+
+from pymongo import MongoClient
+
+load_dotenv()
+
+# staging:   encoach-staging.json
+# prod:      storied-phalanx-349916.json
+
+mongo_db = MongoClient(os.getenv('MONGODB_URI'))[os.getenv('MONGODB_DB')]
+
+if __name__ == "__main__":
+    with open('pathways_2_rw.json', 'r', encoding='utf-8') as file:
+        book = json.load(file)
+
+    tips = []
+    for unit in book["units"]:
+        for page in unit["pages"]:
+            for tip in page["tips"]:
+                new_tip = {
+                    "id": tip["id"],
+                    "standalone": tip["standalone"],
+                    "tipCategory": tip["category"],
+                    "tipHtml": tip["html"]
+                }
+                if not tip["standalone"]:
+                    new_tip["exercise"] = tip["exercise"]
+                tips.append(new_tip)
+
+    for tip in tips:
+        doc_ref = mongo_db.walkthrough.insert_one(tip)

modules/upload_level/__init__.py

@@ -0,0 +1,5 @@
+from .service import UploadLevelService
+
+__all__ = [
+    "UploadLevelService"
+]

modules/upload_level/exam_dtos.py

@@ -0,0 +1,57 @@
+from pydantic import BaseModel, Field
+from typing import List, Dict, Union, Optional, Any
+from uuid import uuid4, UUID
+
+
+class Option(BaseModel):
+    id: str
+    text: str
+
+
+class MultipleChoiceQuestion(BaseModel):
+    id: str
+    prompt: str
+    variant: str = "text"
+    solution: str
+    options: List[Option]
+
+
+class MultipleChoiceExercise(BaseModel):
+    id: UUID = Field(default_factory=uuid4)
+    type: str = "multipleChoice"
+    prompt: str = "Select the appropriate option."
+    questions: List[MultipleChoiceQuestion]
+    userSolutions: List = Field(default_factory=list)
+
+
+class FillBlanksWord(BaseModel):
+    id: str
+    options: Dict[str, str]
+
+
+class FillBlanksSolution(BaseModel):
+    id: str
+    solution: str
+
+
+class FillBlanksExercise(BaseModel):
+    id: UUID = Field(default_factory=uuid4)
+    type: str = "fillBlanks"
+    variant: str = "mc"
+    prompt: str = "Click a blank to select the appropriate word for it."
+    text: str
+    solutions: List[FillBlanksSolution]
+    words: List[FillBlanksWord]
+    userSolutions: List = Field(default_factory=list)
+
+
+Exercise = Union[MultipleChoiceExercise, FillBlanksExercise]
+
+
+class Part(BaseModel):
+    exercises: List[Exercise]
+    context: Optional[str] = Field(default=None)
+
+
+class Exam(BaseModel):
+    parts: List[Part]

modules/upload_level/mapper.py

@@ -0,0 +1,66 @@
+from typing import Dict, Any
+
+from pydantic import ValidationError
+
+from modules.upload_level.exam_dtos import (
+    MultipleChoiceExercise,
+    FillBlanksExercise,
+    Part, Exam
+)
+from modules.upload_level.sheet_dtos import Sheet, Option, MultipleChoiceQuestion, FillBlanksWord
+
+
+class ExamMapper:
+
+    @staticmethod
+    def map_to_exam_model(response: Dict[str, Any]) -> Exam:
+        parts = []
+        for part in response['parts']:
+            part_exercises = part['exercises']
+            context = part.get('context', None)
+
+            exercises = []
+            for exercise in part_exercises:
+                exercise_type = exercise['type']
+                if exercise_type == 'multipleChoice':
+                    exercise_model = MultipleChoiceExercise(**exercise)
+                elif exercise_type == 'fillBlanks':
+                    exercise_model = FillBlanksExercise(**exercise)
+                else:
+                    raise ValidationError(f"Unknown exercise type: {exercise_type}")
+
+                exercises.append(exercise_model)
+
+            part_kwargs = {"exercises": exercises}
+            if context is not None:
+                part_kwargs["context"] = context
+
+            part_model = Part(**part_kwargs)
+            parts.append(part_model)
+
+        return Exam(parts=parts)
+
+    @staticmethod
+    def map_to_sheet(response: Dict[str, Any]) -> Sheet:
+        components = []
+
+        for item in response["components"]:
+            component_type = item["type"]
+
+            if component_type == "multipleChoice":
+                options = [Option(id=opt["id"], text=opt["text"]) for opt in item["options"]]
+                components.append(MultipleChoiceQuestion(
+                    id=item["id"],
+                    prompt=item["prompt"],
+                    variant=item.get("variant", "text"),
+                    options=options
+                ))
+            elif component_type == "fillBlanks":
+                components.append(FillBlanksWord(
+                    id=item["id"],
+                    options=item["options"]
+                ))
+            else:
+                components.append(item)
+
+        return Sheet(components=components)

modules/upload_level/service.py

@@ -0,0 +1,385 @@
+import json
+import os
+import uuid
+from logging import getLogger
+
+from typing import Dict, Any, Tuple, Callable
+
+import pdfplumber
+
+from modules import GPT
+from modules.helper.file_helper import FileHelper
+from modules.helper import LoggerHelper
+from modules.upload_level.exam_dtos import Exam
+from modules.upload_level.mapper import ExamMapper
+from modules.upload_level.sheet_dtos import Sheet
+
+
+class UploadLevelService:
+    def __init__(self, openai: GPT):
+        self._logger = getLogger(__name__)
+        self._llm = openai
+
+    def generate_level_from_file(self, file) -> Dict[str, Any] | None:
+        ext, path_id = FileHelper.save_upload(file)
+        FileHelper.convert_file_to_pdf(
+            f'./tmp/{path_id}/uploaded.{ext}', f'./tmp/{path_id}/exercises.pdf'
+        )
+        file_has_images = self._check_pdf_for_images(f'./tmp/{path_id}/exercises.pdf')
+
+        if not file_has_images:
+            FileHelper.convert_file_to_html(f'./tmp/{path_id}/uploaded.{ext}', f'./tmp/{path_id}/exercises.html')
+
+        completion: Callable[[str], Exam] = self._png_completion if file_has_images else self._html_completion
+        response = completion(path_id)
+
+        FileHelper.remove_directory(f'./tmp/{path_id}')
+
+        if response:
+            return self.fix_ids(response.dict(exclude_none=True))
+        return None
+
+    @staticmethod
+    @LoggerHelper.suppress_loggers()
+    def _check_pdf_for_images(pdf_path: str) -> bool:
+        with pdfplumber.open(pdf_path) as pdf:
+            for page in pdf.pages:
+                if page.images:
+                    return True
+        return False
+
+    def _level_json_schema(self):
+        return {
+            "parts": [
+                {
+                    "context": "<this attribute is optional you may exclude it if not required>",
+                    "exercises": [
+                        self._multiple_choice_html(),
+                        self._passage_blank_space_html()
+                    ]
+                }
+            ]
+        }
+
+    def _html_completion(self, path_id: str) -> Exam:
+        with open(f'./tmp/{path_id}/exercises.html', 'r', encoding='utf-8') as f:
+            html = f.read()
+
+        return self._llm.prediction(
+            [self._gpt_instructions_html(),
+             {
+                 "role": "user",
+                 "content": html
+             }
+             ],
+            ExamMapper.map_to_exam_model,
+            str(self._level_json_schema())
+        )
+
+    def _gpt_instructions_html(self):
+        return {
+            "role": "system",
+            "content": (
+                'You are GPT Scraper and your job is to clean dirty html into clean usable JSON formatted data.'
+                'Your current task is to scrape html english questions sheets.\n\n'
+
+                'In the question sheet you will only see 4 types of question:\n'
+                '- blank space multiple choice\n'
+                '- underline multiple choice\n'
+                '- reading passage blank space multiple choice\n'
+                '- reading passage multiple choice\n\n'
+
+                'For the first two types of questions the template is the same but the question prompts differ, '
+                'whilst in the blank space multiple choice you must include in the prompt the blank spaces with '
+                'multiple "_", in the underline you must include in the prompt the <u></u> to '
+                'indicate the underline and the options a, b, c, d must be the ordered underlines in the prompt.\n\n'
+
+                'For the reading passage exercise you must handle the formatting of the passages. If it is a '
+                'reading passage with blank spaces you will see blanks represented with (question id) followed by a '
+                'line and your job is to replace the brackets with the question id and line with "{{question id}}" '
+                'with 2 newlines between paragraphs. For the reading passages without blanks you must remove '
+                'any numbers that may be there to specify paragraph numbers or line numbers, and place 2 newlines '
+                'between paragraphs.\n\n'
+                
+                'IMPORTANT: Note that for the reading passages, the html might not reflect the actual paragraph '
+                'structure, don\'t format the reading passages paragraphs only by the <p></p> tags, try to figure '
+                'out the best paragraph separation possible.'
+
+                'You will place all the information in a single JSON: {"parts": [{"exercises": [{...}], "context": ""}]}\n '
+                'Where {...} are the exercises templates for each part of a question sheet and the optional field '
+                'context.'
+                
+                'IMPORTANT: The question sheet may be divided by sections but you need to only consider the parts, '
+                'so that you can group the exercises by the parts that are in the html, this is crucial since only '
+                'reading passage multiple choice require context and if the context is included in parts where it '
+                'is not required the UI will be messed up. Some make sure to correctly group the exercises by parts.\n'
+                
+                'The templates for the exercises are the following:\n'
+                '- blank space multiple choice, underline multiple choice and reading passage multiple choice: '
+                f'{self._multiple_choice_html()}\n'
+                f'- reading passage blank space multiple choice: {self._passage_blank_space_html()}\n'
+                
+                'IMPORTANT: For the reading passage multiple choice the context field must be set with the reading '
+                'passages without paragraphs or line numbers, with 2 newlines between paragraphs, for the other '
+                'exercises exclude the context field.'
+            )
+        }
+
+    @staticmethod
+    def _multiple_choice_html():
+        return {
+            "type": "multipleChoice",
+            "prompt": "Select the appropriate option.",
+            "questions": [
+                {
+                    "id": "<the question id>",
+                    "prompt": "<the question>",
+                    "solution": "<the option id solution>",
+                    "options": [
+                        {
+                            "id": "A",
+                            "text": "<the a option>"
+                        },
+                        {
+                            "id": "B",
+                            "text": "<the b option>"
+                        },
+                        {
+                            "id": "C",
+                            "text": "<the c option>"
+                        },
+                        {
+                            "id": "D",
+                            "text": "<the d option>"
+                        }
+                    ]
+                }
+            ]
+        }
+
+    @staticmethod
+    def _passage_blank_space_html():
+        return {
+            "type": "fillBlanks",
+            "variant": "mc",
+            "prompt": "Click a blank to select the appropriate word for it.",
+            "text": (
+                "<The whole text for the exercise with replacements for blank spaces and their "
+                "ids with {{<question id>}} with 2 newlines between paragraphs>"
+            ),
+            "solutions": [
+                {
+                    "id": "<question id>",
+                    "solution": "<the option that holds the solution>"
+                }
+            ],
+            "words": [
+                {
+                    "id": "<question id>",
+                    "options": {
+                        "A": "<a option>",
+                        "B": "<b option>",
+                        "C": "<c option>",
+                        "D": "<d option>"
+                    }
+                }
+            ]
+        }
+
+    def _png_completion(self, path_id: str) -> Exam:
+        FileHelper.pdf_to_png(path_id)
+
+        tmp_files = os.listdir(f'./tmp/{path_id}')
+        pages = [f for f in tmp_files if f.startswith('page-') and f.endswith('.png')]
+        pages.sort(key=lambda f: int(f.split('-')[1].split('.')[0]))
+
+        json_schema = {
+            "components": [
+                {"type": "part", "part": "<name or number of the part>"},
+                self._multiple_choice_png(),
+                {"type": "blanksPassage", "text": (
+                    "<The whole text for the exercise with replacements for blank spaces and their "
+                    "ids with {{<question id>}} with 2 newlines between paragraphs>"
+                )},
+                {"type": "passage", "context": (
+                    "<reading passages without paragraphs or line numbers, with 2 newlines between paragraphs>"
+                )},
+                self._passage_blank_space_png()
+            ]
+        }
+
+        components = []
+
+        for i in range(len(pages)):
+            current_page = pages[i]
+            next_page = pages[i + 1] if i + 1 < len(pages) else None
+            batch = [current_page, next_page] if next_page else [current_page]
+
+            sheet = self._png_batch(path_id, batch, json_schema)
+            sheet.batch = i + 1
+            components.append(sheet.dict())
+
+        batches = {"batches": components}
+        with open('output.json', 'w') as json_file:
+            json.dump(batches, json_file, indent=4)
+
+        return self._batches_to_exam_completion(batches)
+
+    def _png_batch(self, path_id: str, files: list[str], json_schema) -> Sheet:
+        return self._llm.prediction(
+            [self._gpt_instructions_png(),
+             {
+                 "role": "user",
+                 "content": [
+                     *FileHelper.b64_pngs(path_id, files)
+                 ]
+             }
+             ],
+            ExamMapper.map_to_sheet,
+            str(json_schema)
+        )
+
+    def _gpt_instructions_png(self):
+        return {
+            "role": "system",
+            "content": (
+                'You are GPT OCR and your job is to scan image text data and format it to JSON format.'
+                'Your current task is to scan english questions sheets.\n\n'
+                
+                'You will place all the information in a single JSON: {"components": [{...}]} where {...} is a set of '
+                'sheet components you will retrieve from the images, the components and their corresponding JSON '
+                'templates are as follows:\n'
+                
+                '- Part, a standalone part or part of a section of the question sheet: '
+                '{"type": "part", "part": "<name or number of the part>"}\n'
+                
+                '- Multiple Choice Question, there are three types of multiple choice questions that differ on '
+                'the prompt field of the template: blanks, underlines and normal. '
+
+                'In the blanks prompt you must leave 5 underscores to represent the blank space. '
+                'In the underlines questions the objective is to pick the words that are incorrect in the given '
+                'sentence, for these questions you must wrap the answer to the question with the html tag <u></u>, '
+                'choose 3 other words to wrap in <u></u>, place them in the prompt field and use the underlined words '
+                'in the order they appear in the question for the options A to D, disreguard options that might be '
+                'included underneath the underlines question and use the ones you wrapped in <u></u>.'
+                'In normal you just leave the question as is. '
+                
+                f'The template for multiple choice questions is the following: {self._multiple_choice_png()}.\n'
+                
+                '- Reading Passages, there are two types of reading passages. Reading passages where you will see '
+                'blanks represented by a (question id) followed by a line, you must format these types of reading '
+                'passages to be only the text with the brackets that have the question id and line replaced with '
+                '"{{question id}}", also place 2 newlines between paragraphs. For the reading passages without blanks '
+                'you must remove any numbers that may be there to specify paragraph numbers or line numbers, '
+                'and place 2 newlines between paragraphs. '
+                
+                'For the reading passages with blanks the template is: {"type": "blanksPassage", '
+                '"text": "<The whole text for the exercise with replacements for blank spaces and their '
+                'ids that are enclosed in brackets with {{<question id>}} also place 2 newlines between paragraphs>"}. '
+                
+                'For the reading passage without blanks is: {"type": "passage", "context": "<reading passages without '
+                'paragraphs or line numbers, with 2 newlines between paragraphs>"}\n'
+                
+                '- Blanks Options, options for a blanks reading passage exercise, this type of component is a group of '
+                'options with the question id and the options from a to d. The template is: '
+                f'{self._passage_blank_space_png()}\n'
+
+                'IMPORTANT: You must place the components in the order that they were given to you. If an exercise or '
+                'reading passages are cut off don\'t include them in the JSON.'
+            )
+        }
+
+    def _multiple_choice_png(self):
+        multiple_choice = self._multiple_choice_html()["questions"][0]
+        multiple_choice["type"] = "multipleChoice"
+        multiple_choice.pop("solution")
+        return multiple_choice
+
+    def _passage_blank_space_png(self):
+        passage_blank_space = self._passage_blank_space_html()["words"][0]
+        passage_blank_space["type"] = "fillBlanks"
+        return passage_blank_space
+
+    def _batches_to_exam_completion(self, batches: Dict[str, Any]) -> Exam:
+        return self._llm.prediction(
+            [self._gpt_instructions_html(),
+             {
+                 "role": "user",
+                 "content": str(batches)
+             }
+             ],
+            ExamMapper.map_to_exam_model,
+            str(self._level_json_schema())
+        )
+
+    def _gpt_instructions_batches(self):
+        return {
+            "role": "system",
+            "content": (
+                'You are helpfull assistant. Your task is to merge multiple batches of english question sheet '
+                'components and solve the questions. Each batch may contain overlapping content with the previous '
+                'batch, or close enough content which needs to be excluded. The components are as follows:'
+
+                '- Part, a standalone part or part of a section of the question sheet: '
+                '{"type": "part", "part": "<name or number of the part>"}\n'
+
+                '- Multiple Choice Question, there are three types of multiple choice questions that differ on '
+                'the prompt field of the template: blanks, underlines and normal. '
+
+                'In a blanks question, the prompt has underscores to represent the blank space, you must select the '
+                'appropriate option to solve it.'
+                
+                'In a underlines question, the prompt has 4 underlines represented by the html tags <u></u>, you must '
+                'select the option that makes the prompt incorrect to solve it. If the options order doesn\'t reflect '
+                'the order in which the underlines appear in the prompt you will need to fix it.'
+                
+                'In a normal question there isn\'t either blanks or underlines in the prompt, you should just '
+                'select the appropriate solution.'
+
+                f'The template for these questions is the same: {self._multiple_choice_png()}\n'
+
+                '- Reading Passages, there are two types of reading passages with different templates. The one with '
+                'type "blanksPassage" where the text field holds the passage and a blank is represented by '
+                '{{<some number>}} and the other one with type "passage" that has the context field with just '
+                'reading passages. For both of these components you will have to remove any additional data that might '
+                'be related to a question description and also remove some "(<question id>)" and "_" from blanksPassage'
+                ' if there are any. These components are used in conjunction with other ones.'
+
+                '- Blanks Options, options for a blanks reading passage exercise, this type of component is a group of '
+                'options with the question id and the options from a to d. The template is: '
+                f'{self._passage_blank_space_png()}\n\n'
+                
+                'Now that you know the possible components here\'s what I want you to do:\n'
+                '1. Remove duplicates. A batch will have duplicates of other batches and the components of '
+                'the next batch should always take precedence over the previous one batch, what I mean by this is that '
+                'if batch 1 has, for example, multiple choice question with id 10 and the next one also has id 10, '
+                'you pick the next one.\n'
+                '2. Solve the exercises. There are 4 types of exercises, the 3 multipleChoice variants + a fill blanks '
+                'exercise. For the multiple choice question follow the previous instruction to solve them and place '
+                f'them in this format: {self._multiple_choice_html()}. For the fill blanks exercises you need to match '
+                'the correct blanksPassage to the correct fillBlanks options and then pick the correct option. Here is '
+                f'the template for this exercise: {self._passage_blank_space_html()}.\n'
+                f'3. Restructure the JSON to match this template: {self._level_json_schema()}. You must group the exercises by '
+                'the parts in the order they appear in the batches components. The context field of a part is the '
+                'context of a passage component that has text relevant to normal multiple choice questions.\n'
+                
+                'Do your utmost to fullfill the requisites, make sure you include all non-duplicate questions'
+                'in your response and correctly structure the JSON.'
+            )
+        }
+
+    @staticmethod
+    def fix_ids(response):
+        counter = 1
+        for part in response["parts"]:
+            for exercise in part["exercises"]:
+                if exercise["type"] == "multipleChoice":
+                    for question in exercise["questions"]:
+                        question["id"] = counter
+                        counter += 1
+                if exercise["type"] == "fillBlanks":
+                    for i in range(len(exercise["words"])):
+                        exercise["words"][i]["id"] = counter
+                        exercise["solutions"][i]["id"] = counter
+                        counter += 1
+        return response

modules/upload_level/sheet_dtos.py

@@ -0,0 +1,29 @@
+from pydantic import BaseModel
+from typing import List, Dict, Union, Any, Optional
+
+
+class Option(BaseModel):
+    id: str
+    text: str
+
+
+class MultipleChoiceQuestion(BaseModel):
+    type: str = "multipleChoice"
+    id: str
+    prompt: str
+    variant: str = "text"
+    options: List[Option]
+
+
+class FillBlanksWord(BaseModel):
+    type: str = "fillBlanks"
+    id: str
+    options: Dict[str, str]
+
+
+Component = Union[MultipleChoiceQuestion, FillBlanksWord, Dict[str, Any]]
+
+
+class Sheet(BaseModel):
+    batch: Optional[int] = None
+    components: List[Component]

text-samples/c-ielts-1.csv

@@ -1,5 +0,0 @@
-QUESTION,ANSWER,GRADE
-"News editors decide what to broadcast on television and what to print in newspapers. What factors do you think influence these decisions? Do we become used to bad news? Would it he better if more good news was reported?","It has often been said that “Good news is bad news” because it does not sell newspapers. A radio station that once decided to present only good news soon found that it had gone out of business for lack of listeners. Bad news on the other hand is so common that in order to cope with it, we often simply ignore it. We have become immune to bad news and the newspapers and radio stations are aware of this. While newspapers and TV stations may aim to report world events accurately, be they natural or human disasters, political events or the horrors of war, it is also true that their main objective is to sell newspapers and attract listeners and viewers to their stations. For this reason TV and radio stations attempt to reflect the flavour of their station by providing news broadcasts tailor-made to suit their listeners’ preferences. Programmes specialising in pop music or TV soap operas focus more on local news, home issues and up-to-date traffic reports. The more serious stations and newspapers like to provide “so called” objective news reports with editorial comment aimed at analysing the situation. If it is true, then, that newspapers and TV stations are tailoring their news to their readers’ and viewers’ requirements, how can they possibly be reporting real world events in an honest and objective light? Many radio and TV stations do, in fact, report items of good news but they no longer call this news. They refer to these as human interest stories and package them in programmes specialising, for instance, in consumer affairs or local issues. Good news now comes to us in the form of documentaries the fight against children’s cancer or AIDS, or the latest developments in the fight to save the planet from environmental pollution.",6
-"We are becoming increasingly dependent on computers. They are used in businesses, hospitals, crime detection and even to fly planes. What things will they be used for in the future? Is this dependence on computers a good thing or should we he more auspicious of their benefits?","Computers are a relatively new invention. The first computers were built fifty years ago and it is only in the last thirty or so years that their influence has affected our everyday life. Personal computers were introduced as recently as the early eighties. In this short time they have made a tremendous impact on our lives. We are now so dependent on computers that it is hard to imagine what things would be like today without them. You have only got to go into a bank when their main computer is broken to appreciate the chaos that would occur if computers were suddenly removed world-wide. In the future computers will be used to create bigger and even more sophisticated computers. The prospects for this are quite alarming. They will be so complex that no individual could hope to understand how they work. They will bring a lot of benefits but they will also increase the potential for unimaginable chaos. They will, for example, be able to fly planes and they will be able to co ordinate the movements of several planes in the vicinity of an airport. Providing all the computers are working correctly nothing can go wrong. If one small program fails — disaster. There is a certain inevitability that technology will progress and become increasingly complex. We should, however, ensure that we are still in a position where we are able to control technology. It will be all too easy to suddenly discover that technology is controlling us. By then it might be too late I believe that it is very important to be suspicious of the benefits that computers will bring and to make sure that we never become totally dependent on a completely technological world.",6
-"The chart below shows the amount of money per week spent on fast foods in Britain. The graph shows the trends in consumption of fast-foods. Write a report for a university lecturer describing the information shown below.","The chart shows that high income earners consumed considerably more fast foods than the other income groups, spending more than twice as much on hamburgers (43 pence per person per week) than on fish and chips or pizza (both under 20 pence). Average income earners also favoured hamburgers, spending 33 pence per person per week, followed by fish and chips at 24 pence, then pizza at 11 pence. Low income earners appear to spend less than other income groups on fast foods, though fish and chips remains their most popular fast food, followed by hamburgers and then pizza. From the graph we can see that in 1970, fish and chips were twice as popular as burgers, pizza being at that time the least popular fast food. The consumption of hamburgers and pizza has risen steadily over the 20 year period to 1990 while the consumption of fish and chips has been in decline over that same period with a slight increase in popularity since 1985.",7.5
-"You have had a bank account for a few years. Recently you received a letter from the hank stating that your account is $240 overdrawn and that you will he charged $70 which will he taken directly from your account. You know that this information is incorrect. Write a letter to the bank. Explain what has happened and say what you would like them to do about it.","Dear Sir, I am writing in reply to a letter I received from you a few days ago. In your letter you state that I am $240 overdrawn and that you will be charging me $70.I would like to point out that the reason I am overdrawn is because of a mistake made by your bank. If you look through your records you will see that I wrote several weeks ago explaining the situation. For the last twelve months, I have been paying $300 a month for a car I bought last summer. The monthly payments were taken directly from my bank account. However, two months ago I sold the car and I wrote to you instructing you to stop paying the monthly instalments. I received a letter from you acknowledging my request, but, for some reason, nothing was done about it. Another $300 instalment has been paid this month and this is the reason why I am overdrawn. I would like you to contact the garage where I bought the car explaining your error. I would also like you to ask them to return the money. Yours faithfully, P Stoft",8

text-samples/c-ielts-2.csv

@@ -1,5 +0,0 @@
-QUESTION,ANSWER,GRADE
-"The table below shows the consumer durables (telephone, refrigerator, etc.) owned in Britain from 1972 to 1983. Write a report for a university lecturer describing the information shown below","The chart shows that the percentage of British households with a range of consumer durables steadily in- creased between 1972 and 1983. The greatest increase was in telephone ownership, rising from 42% in 1972 to 77% in 1983. Next came central heating ownership, rising from 37% of households in 1972 to 64% in 1983. The percentage of households with a refrigerator rose 21% over the same period and of those with a washing machine by 14%. Households with vacuum-cleaners, televisions and dishwash- ers increased by 8%, 5% and 2% respectively. In 1983, the year of their introduction, 18% of households had a video recorder. The significant social changes reflected in the statistics are that over the period the proportion of British houses with central heating rose from one to two thirds, and of those with a phone from under a half to over three-quarters. Together with the big increases in the ownership of washing machines and refriger- ators, they are evidence of both rising living standards and the trend to lifestyles based on comfort and convenience.",8
-"Fatherhood ought to be emphasised as much as motherhood. The idea that women are solely responsible for deciding whether or not to have babies leads on to the idea that they are also responsible for bringing the children up. To what extent do you agree or disagree?","I believe that child-rearing should be the responsibility of both parents and that, whilst the roles within that partnership may be different, they are nevertheless equal in importance. In some societies, it has been made easier over the years for single parents to raise children on their own. However, this does not mean that the traditional family, with both parents providing emotional support and role-models for their children, is not the most satisfactory way of bringing up children. Of crucial importance, in my opinion, is how we define 'responsible for bringing the children up'. At its simplest, it could mean giving the financial support necessary to provide a home, food and clothes and making sure the child is safe and receives an adequate education. This would be the basic definition. There is, however, another possible way of defining that part of the quotation. That would say it is not just the father's responsibility to provide the basics for his children, while his wife involves herself in the everyday activity of bringing them up. Rather, he should share those daily duties, spend as much time as his job allows with his children, play with them, read to them, help directly with their educa- tion, participate very fully in their lives and encourage them to share his. It is this second, fuller, concept of 'fatherhood' that I am in favour of, although I also realise how difficult it is to achieve sometimes. The economic and employment situation in many countries means that jobs are getting more, not less, stressful, requiring long hours and perhaps long journeys to work as well. Therefore it may remain for many a desirable ideal rather than an achievable reality.",8
-"The chart below shows the amount of leisure time enjoyed by men and women of different employment status. Write a report for a university lecturer describing the information shown below.","The chart shows the number of hours of leisure enjoyed by men and women in a typical week in 1998-9, according to gender and employment status. Among those employed full-time, men on average had fifty hours of leisure, whereas women had ap- proximately thirty-seven hours. There were no figures given for male part-time workers, but female part-timers had forty hours of leisure time, only slightly more than women in full-time employment. perhaps reflecting their work in the home. In the unemployed and retired categories, leisure time showed an increase for both sexes, as might have been expected. Here too, men enjoyed more leisure time-over eighty hours, compared with seventy hours for women, perhaps once again reflecting the fact that women spend more time working in the home than men. Lastly, housewives enjoyed approximately fifty-four hours of leisure, on average. There were no figures given for househusbands! Overall, the chart demonstrates that in the categories for which statistics on male leisure time were available, men enjoyed at least ten hours of extra leisure time.",6
-"Prevention is better than cure. Out of a country's health budget, a large proportion should be diverted from treatment to spending on health education and preventative measures. To what extent do you agree or disagree with this statement?","Of course it goes without saying that prevention is better than cure. That is why, in recent years, there has been a growing body of opinion in favour of putting more resources into health education and preventive measures. The argument is that ignorance of, for example, basic hygiene or the dangers of an unhealthy diet or lifestyle needs to be combatted by special nationwide publicity campaigns, as well as longer-term health education. Obviously, there is a strong human argument for catching any medical condition as early as possible. There is also an economic argument for doing so. Statistics demonstrate the cost-effectiveness of treat- ing a condition in the early stages, rather than delaying until more expensive and prolonged treatment is necessary. Then there are social or economic costs, perhaps in terms of loss of earnings for the family concerned or unemployed benefit paid by the state. So far so good, but the difficulties start when we try to define what the 'proportion of the budget should be, particularly if the funds will be diverted from treatment. Decisions on exactly how much of the total health budget should be spent in this way are not a matter for the non-specialist, but should be made on the basis of an accepted health service model. This is the point at which real problems occur the formulation of the model. How do we accurately measure which health education campaigns are effective in both medical and financial terms? How do we agree about the medical efficacy of various screening programmes, for example, when the medical establishment itself does not agree? A very rigorous process of evaluation is called for, so that we can make informed decisions.",6.5

text-samples/ielts-idp-com.csv

@@ -1,6 +0,0 @@
-QUESTION,ANSWER,GRADE,COMMENT
-"You live in a room in college which you share with another student. However, there are many problems with this arrangement and you find it very difficult to work. Write a letter to the accommodation officer at the college. In your letter describe the situation, explain your problems and why it is difficult to work, say what kind of accommodation you would prefer","Dear Sir/Madam, I am writing to express my dissatisfaction with my room-mate. As you know we share one room, I can not study in the room at all any more if I still stay there. She always has friend visiting and has parties in the room. They make lots of noise and switch on the radio very loudly, for me this environment is very difficult to study and I need a quiet room. Even borrows my things without asking, it is very impolite. I request you can give me a new room next term because I have been asked her has parties in other place many times they still have parties in the room. I really can not stay in the same room with her. I would be grateful if you could change me a single room. Your faithfully, Catherine",5,"The answer is below the word limit and there is some repetition of the task rubric. (Length is a common problem in General Training scripts.) Answers that are short lose marks because of inadequate content and may also lose marks because there is insufficient material in the answer for the examiner to give credit for accuracy and coherence. Despite these problems, the introduction to the letter is appropriate and the purpose of the writer is clear. The points are not always linked together well and punctuation is sometimes faulty. The sentences are kept quite simple and mistakes occur as soon as more complex structures are attempted."
-"In Britain, when someone gets old they often go to live in a home with other old people where there are nurses to look after them. Sometimes the government has to pay for this care.","Who should be responsible for our people. It is true that the old Peoples situation gets worse in the many countries. The first question must be what they want’s and what they needs? Especially their necessity are more benefit more respect more quiet life. If they have been working for a long time in the any company or in the Public Sector and when they get old that’s means during their retire’s time company or Government must be responsible of their welfare, it is just my opinion. They should take care of them. In addition to company or Government. If they have good money they can look after themselves. We can do something to make easier their life for example an organization or a voluntary association, unions. The families or Relative’s responsibility depends on their wealthy situations. If they could do they should do anything. Government’s or their former place could supply them with life insurance and a good Social Security Policy. The Social community center or old age pensioner like in the Britain are very useful for them. For all of them life is hard and gets harder, in the their old ages. They expect more attention and good life. The old people, if don’t want lost them. We should do anything that what we able to do.",5,"There are quite a lot of relevant ideas in the answer but they are not always well supported and sometimes they are unclear.There are some areas in the answer where the organisation becomes weak and the reader finds the message difficult to follow. Nevertheless, the writer’s view is apparent and there is a logical flow to th points given. There are a lot of mistakes in the answer and some parts, such as the conclusion, are very hard to follow because of these errors. Although there is some appropriate vocabulary, sentence control is very weak."
-"These days, more and more people move away from the area where they were born and brought up when they become adults. Do the advantages of this development outweigh the disadvantages?","It is certainly the case in my local area that many young people choose to leave their home village or town as soon as they finish college or when they first get full-time employment. There are several advantages to this. Firstly, it gives the individuals better opportunities to find more suitable jobs. This means they have much greater flexibility in the careers they can choose and are no longer forced to take the work available in the local area. A second benefit is that they have the chance to meet and work alongside a wider variety of people, which enriches their social and professional lives. Another relevant point is that moving to a place where they are anonymous allows people greater freedom to behave as they wish, without worrying about what those around them think. However, there are a number of drawbacks to this development, the most serious being loss of support. It is important for humans to feel that they are part of a community and can rely on family and friends for help, on a day-to-day basis. In a place where individuals know few people it is easy to become isolated and lonely. Related to this point is the fact that when people know very little about their neighbours, it is hard for mutual trust to develop. When people have lived in the same place or village all their lives, their personal and family backgrounds are widely known and this information can help others make reliable judgements, building personal and business relationships. On balance, I feel that this trend brings more negative outcomes than advantages and that it is leading to real problems of isolation and erosion of identity",6.5,"Overall, your answer demonstrates a clear understanding of the question and presents relevant ideas and arguments. Your essay structure is well-organized with an introduction, body paragraphs, and a conclusion. You have effectively addressed both the advantages and disadvantages of people moving away from their birthplace when they become adults."
-"The average standard of people's health is likely to be lower in the future than it is now. To what extent do you agree or disagree with this statement?","I completly disagree with the written statment. I believe that most of the people in the world have more information about their health and also about how they can improve their healthy conditions. Nowadays, information about how harmful is to smoke for our bodies can be seen in many packets of cigars. This is a clear example how things can change from our recent past. There is a clear trend in the diminishing of smokers and if this continues it will have a positive impact in our health. On the other hand, the alimentation habbits are changing all over the world and this can affect people’s health. However every one can choose what to eat every day. Mostly everybody, from developed societies, know the importance of having a healthy diet. Advances such as the information showed in the menus of fast food restaurants will help people to have a clever choice before they choose what to eat. Another important issue that I would like to mention is how medicine is changing. There are new discovers and treatments almost every week and that is an inequivoque sintom of how things are changing in order to improve the world’s health.",5.5,"A clear position is presented from the outset, supported by relevant ideas. These would require further development to achieve a higher score. The response is under-length, however. Information and ideas are generally arranged coherently and there is a clear overall progression. Cohesive devices are used effectively, but paragraphing is not always logical. A range of vocabulary is attempted, although there are some errors in spelling, word choice and word formation. There also appears to be some interference from the test taker’s first language, e.g. ‘alimentation’, but these features do not make the answer difficult to understand. There is a mix of sentence forms, but the level of error is too high to achieve a higher band score."
-"The average standard of people's health is likely to be lower in the future than it is now. To what extent do you agree or disagree with this statement?","Recently, there have been a lot of discussions about health and whether it is going to improve or not. In my opinion, I think that people will become unhealthier in the future than they are now. There are many reasons that support the idea of people becoming unhealthy in the future. Firstly, one reason is that of food. People tend to eat more fast food nowadays. They tend to treat themselves with sweets and chocolate whenever they want. This appears to be because people are busier now than they used to be. So, people don’t have a chance to cook or even learn the art of cookery. Also, having a lot of unhealthy food can lead to obesity and it could be a serious issue in the future. Another reason is that technology is developing everyday. Young people enjoy buying new gadgets and the latest devices. This has a negative impact on their health, especially when they enjoy video games. Spending long hours looking at a screen can lead to bad eyesight and obesity as well. Yet another reason is that laziness is a big issue. Different forms of exercise might disappear in the future because people don’t like sports. Also, people prefer spending most of their time on the internet and the internet is growing every single day. Other people might disagree and say that health will improve in the future. They believe that new sports and new ways to exercise will appear in the future. However, I don’t think it can happen since the majority of people spend less time outdoors. Moreover, other people believe that technology will try and help people improve their health. For example, there have been some games released on the Wii console that makes people exercise but technology is developing more in a negative way. For instance, many phone industries are developing new applications everyday and today’s generation likes to follow every trend. This prevents people to go outside to exercise. They like to spend more time on the internet downloading new programmes or reading gossips about celebraties. This affects people’s health badly. In conclusion, I believe that people’s health is affected negatively by fast food, technology and sports and it will be a problem in the future.",7.5,"The test taker presents a clear position at the outset and explores some ideas to support this. An alternative position is also considered, but rejected. This is a strong response, but there is rather too much emphasis on technology: other aspects of the proposition could also be considered, e.g. less physical work and more sedentary work, greater reliance on cars meaning less exercise, aging populations in some countries leading to more complex health issues. Ideas are organised logically and there is a clear progression throughout the response, with good use of cohesive devices and logical paragraphing. The response could perhaps be improved by breaking down paragraphs 2 and 3. There is a wide range of vocabulary with good use of less common items as well as evidence of higher level features, such as ‘softening’, e.g. ‘They tend to’, ‘This appears to be’, and ‘might disagree’. Errors in spelling and word formation are rare. There is also a variety of complex structures with frequent error-free sentences, though some errors do occur and there is some overuse of rather short sentence forms."

text-samples/listening.txt

@@ -1 +0,0 @@
-"Provide me with an transcript similar to the ones in ielts exam Listening Section 1. After the transcript, please generate a 'form like' fill in the blanks exercise with 6 items (ex: name, date of birth) to fill related to the text content. Finally provide the answers for the exercise. The response must be a json following this format: { 'type': '<type of registration (ex: hotel, gym, english course, etc)>', 'transcript': '<transcript of just the conversation about a registration of some sort, identify the person talking in each speech line>', 'exercise': { 'item': { '1': '<item 1>', '2': '<item 2>', '3': '<item 3>', '4': '<item 4>', '5': '<item 5>', '6': '<item 5>' }, 'answers': {'1': '<answer to fill blank space in item 1>', '2': '<answer to fill blank space in item 2>', '3': '<answer to fill blank space in item 3>', '4': '<answer to fill blank space in item 4>', '5': '<answer to fill blank space in item 5>', '6': '<answer to fill blank space in item 6>'}}}

text-samples/speaking.txt

@@ -1,5 +0,0 @@
-PT-1: How do you usually spend your weekends? Why?
-A: audio-samples/weekends.m4a
-
-PT-2: Describe someone you know who does something well. You should say who this person is, how do you know this person, what they do well and explain why you think this person is so good at doing this.
-A: audio-samples/speakingpt2.m4a

text-samples/writing-pt2.txt

@@ -1,7 +0,0 @@
-Q: It is important for children to learn the difference between right and wrong at an early age. Punishment is necessary to help them learn this distinction. To what extent do you agree or disagree with this opinion? What sort of punishment should parents and teachers be allowed to use to teach good behaviour to children?
-
-
-A: In today's world, moral values and ethics play a vital role in shaping the character of an individual. Children are the building blocks of society, and it is important to inculcate a sense of right and wrong in them from an early age. While punishment can be an effective tool in teaching the difference between right and wrong, it should not be the only approach. In my opinion, punishment should be used in moderation, and parents and teachers should focus on positive reinforcement and guidance to teach good behavior.
-Punishment can be used to correct behavior and to help children understand the consequences of their actions. However, excessive punishment can be counterproductive and can even have harmful effects on children. Physical punishment, such as hitting or spanking, should be avoided as it can lead to physical and emotional trauma. Instead, parents and teachers should consider alternative forms of punishment such as time-outs, loss of privileges or extra chores. These methods can be effective in conveying the message without causing physical harm.
-Furthermore, parents and teachers should focus on positive reinforcement to encourage good behavior. Praising children when they exhibit good behavior can be an effective way to motivate them to continue behaving well. Teachers can use stickers or small rewards to encourage students to work hard and behave well in class. Parents can use similar methods at home to reinforce good behavior.
-In conclusion, it is important for children to learn the difference between right and wrong at an early age. Punishment can be a useful tool in teaching this distinction, but it should not be the only approach. Parents and teachers should use positive reinforcement and guidance to encourage good behavior, and should only resort to punishment in moderation and when necessary. Any form of punishment should be non-violent and should not cause physical or emotional harm to the child.

tmp/placeholder.txt

@@ -0,0 +1 @@
+THIS FILE ONLY EXISTS TO KEEP THIS FOLDER IN THE REPO