Merged in grading-summary (pull request #7)

Grading Summary Endpoint Logic

Approved-by: Tiago Ribeiro
Approved-by: Cristiano Ferreira

app.py

@@ -10,7 +10,7 @@ from helper.firebase_helper import *
 from helper.heygen_api import create_videos_and_save_to_db
 from helper.speech_to_text_helper import *
 from helper.token_counter import count_tokens
-from helper.openai_interface import make_openai_call, make_openai_instruct_call
+from helper.openai_interface import *
 import os
 import re
 import logging
@@ -37,6 +37,7 @@ thread_event = threading.Event()
 logging.basicConfig(level=logging.DEBUG,  # Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
                     format='%(asctime)s - %(levelname)s - %(message)s')
 
+
 @app.route('/healthcheck', methods=['GET'])
 def healthcheck():
     return {"healthy": True}
@@ -407,10 +408,10 @@ def grade_speaking_task_2():
                                       "Speaking Part 2 question: '" + question + "'")
             token_count = count_tokens(perfect_answer_message)["n_tokens"]
             response['perfect_answer'] = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT,
-                                                                  perfect_answer_message,
+                                                                   perfect_answer_message,
-                                                                  token_count,
+                                                                   token_count,
-                                                                  None,
+                                                                   None,
-                                                                  GEN_QUESTION_TEMPERATURE)
+                                                                   GEN_QUESTION_TEMPERATURE)
             return response
         else:
             return {
@@ -495,15 +496,15 @@ def grade_speaking_task_3():
                                       "Speaking Part 3 question: '" + item["question"] + "'")
             token_count = count_tokens(perfect_answer_message)["n_tokens"]
             perfect_answers.append(make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT,
-                                                                   perfect_answer_message,
+                                                             perfect_answer_message,
-                                                                   token_count,
+                                                             token_count,
-                                                                   None,
+                                                             None,
-                                                                   GEN_QUESTION_TEMPERATURE))
+                                                             GEN_QUESTION_TEMPERATURE))
         message = (
-                "Grade this Speaking Part 3 answer according to ielts grading system and provide "
+            "Grade this Speaking Part 3 answer according to ielts grading system and provide "
-                "an elaborated comment where you deep dive into what is wrong and right about the answers."
+            "an elaborated comment where you deep dive into what is wrong and right about the answers."
-                "Please assign a grade of 0 if the answer provided does not address the question."
+            "Please assign a grade of 0 if the answer provided does not address the question."
-                "\n\n The questions and answers are: \n\n'")
+            "\n\n The questions and answers are: \n\n'")
 
         formatted_text = ""
         for i, entry in enumerate(answers, start=1):
@@ -511,9 +512,10 @@ def grade_speaking_task_3():
             formatted_text += f"**Answer {i}:**\n{entry['answer']}\n\n"
 
         message += formatted_text
-        message += ("'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', "
+        message += (
-                    "'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, "
+            "'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', "
-                    "'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}")
+            "'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, "
+            "'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}")
 
         token_count = count_tokens(message)["n_tokens"]
         response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count,
@@ -690,5 +692,18 @@ def fetch_answer_tips():
         return str(e)
 
 
+@app.route('/grading_summary', methods=['POST'])
+@jwt_required()
+def grading_summary():
+    # Body Format
+    # {'sections': Array of {'code': key, 'name': name, 'grade': grade}}
+    # Output Format
+    # {'sections': Array of {'code': key, 'name': name, 'grade': grade, 'evaluation': evaluation, 'suggestions': suggestions}}
+    try:
+        return calculate_grading_summary(request.get_json())
+    except Exception as e:
+        return str(e)
+
+
 if __name__ == '__main__':
     app.run()

helper/openai_interface.py

@@ -16,6 +16,36 @@ TRY_LIMIT = 1
 
 try_count = 0
 
+# GRADING SUMMARY
+chat_config = {'max_tokens': 1000, 'temperature': 0.2}
+section_keys = ['reading', 'listening', 'writing', 'speaking', 'level']
+grade_top_limit = 9
+
+tools = [{
+    "type": "function",
+    "function": {
+        "name": "save_evaluation_and_suggestions",
+        "description": "Saves the evaluation and suggestions requested by input.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "evaluation": {
+                    "type": "string",
+                    "description": "A comment on the IELTS section grade obtained in the specific section and what it could mean without suggestions.",
+                },
+                "suggestions": {
+                    "type": "string",
+                    "description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.",
+                },
+            },
+            "required": ["evaluation", "suggestions"],
+        },
+    }
+}]
+
+
+###
+
 def process_response(input_string, quotation_check_field):
     if '{' in input_string:
         try:
@@ -44,6 +74,7 @@ def process_response(input_string, quotation_check_field):
     else:
         return input_string
 
+
 def parse_string(to_parse: str):
     parsed_string = to_parse.replace("\"", "\\\"")
     pattern = r"(?<!\w)'|'(?!\w)"
@@ -80,6 +111,7 @@ def parse_string_2(to_parse: str):
     to_parse = to_parse.replace(":", ": ")
     return to_parse
 
+
 def remove_special_chars_and_escapes(input_string):
     parsed_string = input_string.replace("\\\"", "'")
     parsed_string = parsed_string.replace("\n\n", " ")
@@ -122,6 +154,7 @@ def make_openai_call(model, messages, token_count, fields_to_check, temperature)
         try_count = 0
         return processed_response
 
+
 def make_openai_instruct_call(model, message: str, token_count, fields_to_check, temperature):
     global try_count
     response = openai.Completion.create(
@@ -141,3 +174,58 @@ def make_openai_instruct_call(model, message: str, token_count, fields_to_check,
     else:
         try_count = 0
         return processed_response
+
+
+# GRADING SUMMARY
+def calculate_grading_summary(body):
+    extracted_sections = extract_existing_sections_from_body(body, section_keys)
+
+    ret = []
+
+    for section in extracted_sections:
+        openai_response_dict = calculate_section_grade_summary(section)
+        ret = ret + [{'code': section['code'], 'name': section['name'], 'grade': section['grade'],
+                      'evaluation': openai_response_dict['evaluation'],
+                      'suggestions': openai_response_dict['suggestions']}]
+
+    return {'sections': ret}
+
+
+def calculate_section_grade_summary(section):
+    res = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        max_tokens=chat_config['max_tokens'],
+        temperature=chat_config['temperature'],
+        tools=tools,
+        messages=[
+            {
+                "role": "user",
+                "content": "You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the grade obtained in the section. You should offer a comment on this grade with also suggestions on how to possibly get a better grade.",
+            },
+            {
+                "role": "user",
+                "content": "Section: " + str(section['name']) + " Grade: " + str(section['grade']),
+            },
+            {"role": "user", "content": "Speak in third person."},
+            {"role": "user", "content": "Please save the evaluation and suggestions generated."}
+        ])
+
+    return parse_openai_response(res)
+
+
+def parse_openai_response(response):
+    if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][
+        0] and 'tool_calls' in response['choices'][0]['message'] and isinstance(
+        response['choices'][0]['message']['tool_calls'], list) and len(
+        response['choices'][0]['message']['tool_calls']) > 0 and \
+            response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
+        return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
+    else:
+        return {'evaluation': "", 'suggestions': ""}
+
+
+def extract_existing_sections_from_body(my_dict, keys_to_extract):
+    if 'sections' in my_dict and isinstance(my_dict['sections'], list) and len(my_dict['sections']) > 0:
+        return list(filter(
+            lambda item: 'code' in item and item['code'] in keys_to_extract and 'grade' in item and 'name' in item,
+            my_dict['sections']))

postman/ielts.postman_collection.json

@@ -1,9 +1,9 @@
 {
 	"info": {
-		"_postman_id": "1b901158-4228-426a-9c96-8cedc4df8470",
+		"_postman_id": "9905f8e4-f3b9-45e4-8ede-434c5de11eca",
 		"name": "ielts",
 		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-		"_exporter_id": "26107457"
+		"_exporter_id": "29491168"
 	},
 	"item": [
 		{
@@ -1104,6 +1104,53 @@
 				}
 			},
 			"response": []
+		},
+		{
+			"name": "Get Grading Summary",
+			"request": {
+				"auth": {
+					"type": "bearer",
+					"bearer": [
+						{
+							"key": "token",
+							"value": "{{jwt_token}}",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [
+					{
+						"key": "Content-Type",
+						"value": "application/json",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"question\": \"When did Kendrick Lamar sign for TDE?\",\n    \"answer\": \"Hello GPT.\",\n\t\t\"correct_answer\": \"2005\",\n    \"context\": \"Kendrick Lamar Duckworth (born June 17, 1987) is an American rapper and songwriter. Known for his progressive musical styles and socially conscious songwriting, he is often considered one of the most influential hip hop artists of his generation. Born and raised in Compton, California, Lamar began his career as a teenager performing under the stage name K.Dot. He quickly garnered local attention which led to him signing a recording contract with Top Dawg Entertainment (TDE) in 2005.\"\n}\n",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "http://127.0.0.1:5000/fetch_tips",
+					"protocol": "http",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "5000",
+					"path": [
+						"fetch_tips"
+					]
+				}
+			},
+			"response": []
 		}
 	]
 }