From f2e84977566a65b1258935308061e328ccd2123d Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Sat, 6 Jan 2024 16:07:46 +0000 Subject: [PATCH 1/6] Added a playground --- module_grading_playground.py | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 module_grading_playground.py diff --git a/module_grading_playground.py b/module_grading_playground.py new file mode 100644 index 0000000..4f7c031 --- /dev/null +++ b/module_grading_playground.py @@ -0,0 +1,94 @@ +import json + +import openai +import os +from dotenv import load_dotenv +from functools import reduce + +load_dotenv() +openai.api_key = os.getenv("OPENAI_API_KEY") + +chat_config = {'max_tokens': 1000, 'temperature': 0.2} +section_keys = ['reading', 'listening', 'writing', 'speaking', 'level'] +grade_top_limit = 9 + +tools = [{ + "type": "function", + "function": { + "name": "save_evaluation_and_suggestions", + "description": "Saves the evaluation and suggestions requested by input.", + "parameters": { + "type": "object", + "properties": { + "evaluation": { + "type": "string", + "description": "A comment on the IELTS section grade obtained in the specific section and what it could mean without suggestions.", + }, + "suggestions": { + "type": "string", + "description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.", + }, + }, + "required": ["evaluation", "suggestions"], + }, + } +}] + + +# Input Format +# {'sections': Array of {'code': key, 'name': name, 'grade': grade}} +# Output +# {'sections': Array of {'code': key, 'name': name, 'grade': grade, 'evaluation': evaluation, 'suggestions': suggestions}} + +def grading_summary(body): + extracted_sections = extract_existing_sections_from_body(body, section_keys) + + ret = [] + + for section in extracted_sections: + openai_response_dict = calculate_section_grade_summary(section) + ret = ret + [{'code': section['code'], 'name': section['name'], 'grade': section['grade'], + 'evaluation': openai_response_dict['evaluation'], + 'suggestions': openai_response_dict['suggestions']}] + + return {'sections': ret} + + +def calculate_section_grade_summary(section): + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + max_tokens=chat_config['max_tokens'], + temperature=chat_config['temperature'], + tools=tools, + messages=[ + { + "role": "user", + "content": "You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the grade obtained in the section. You should offer a comment on this grade with also suggestions on how to possibly get a better grade.", + }, + { + "role": "user", + "content": "Section: " + str(section['name']) + " Grade: " + str(section['grade']), + }, + {"role": "user", "content": "Speak in third person."}, + {"role": "user", "content": "Please save the evaluation and suggestions generated."} + ]) + + return parse_openai_response(res) + + +def parse_openai_response(response): + if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][ + 0] and 'tool_calls' in response['choices'][0]['message'] and isinstance( + response['choices'][0]['message']['tool_calls'], list) and len( + response['choices'][0]['message']['tool_calls']) > 0 and \ + response['choices'][0]['message']['tool_calls'][0]['function']['arguments']: + return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments']) + else: + return {'evaluation': "", 'suggestions': ""} + + +def extract_existing_sections_from_body(my_dict, keys_to_extract): + if 'sections' in my_dict and isinstance(my_dict['sections'], list) and len(my_dict['sections']) > 0: + return list(filter( + lambda item: 'code' in item and item['code'] in keys_to_extract and 'grade' in item and 'name' in item, + my_dict['sections'])) From ac27239787f193c494431aca632490cec0d5ab4e Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Sat, 6 Jan 2024 18:46:29 +0000 Subject: [PATCH 2/6] Calculate Grading Summary Logic --- app.py | 42 ++++++++++++------- .../grading_summary.py | 8 +--- 2 files changed, 28 insertions(+), 22 deletions(-) rename module_grading_playground.py => grading_summary/grading_summary.py (92%) diff --git a/app.py b/app.py index 643e465..ac22564 100644 --- a/app.py +++ b/app.py @@ -11,6 +11,7 @@ from helper.heygen_api import create_videos_and_save_to_db from helper.speech_to_text_helper import * from helper.token_counter import count_tokens from helper.openai_interface import make_openai_call, make_openai_instruct_call +from grading_summary.grading_summary import calculate_grading_summary import os import re import logging @@ -37,6 +38,7 @@ thread_event = threading.Event() logging.basicConfig(level=logging.DEBUG, # Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) format='%(asctime)s - %(levelname)s - %(message)s') + @app.route('/healthcheck', methods=['GET']) def healthcheck(): return {"healthy": True} @@ -407,10 +409,10 @@ def grade_speaking_task_2(): "Speaking Part 2 question: '" + question + "'") token_count = count_tokens(perfect_answer_message)["n_tokens"] response['perfect_answer'] = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, - perfect_answer_message, - token_count, - None, - GEN_QUESTION_TEMPERATURE) + perfect_answer_message, + token_count, + None, + GEN_QUESTION_TEMPERATURE) return response else: return { @@ -495,15 +497,15 @@ def grade_speaking_task_3(): "Speaking Part 3 question: '" + item["question"] + "'") token_count = count_tokens(perfect_answer_message)["n_tokens"] perfect_answers.append(make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, - perfect_answer_message, - token_count, - None, - GEN_QUESTION_TEMPERATURE)) + perfect_answer_message, + token_count, + None, + GEN_QUESTION_TEMPERATURE)) message = ( - "Grade this Speaking Part 3 answer according to ielts grading system and provide " - "an elaborated comment where you deep dive into what is wrong and right about the answers." - "Please assign a grade of 0 if the answer provided does not address the question." - "\n\n The questions and answers are: \n\n'") + "Grade this Speaking Part 3 answer according to ielts grading system and provide " + "an elaborated comment where you deep dive into what is wrong and right about the answers." + "Please assign a grade of 0 if the answer provided does not address the question." + "\n\n The questions and answers are: \n\n'") formatted_text = "" for i, entry in enumerate(answers, start=1): @@ -511,9 +513,10 @@ def grade_speaking_task_3(): formatted_text += f"**Answer {i}:**\n{entry['answer']}\n\n" message += formatted_text - message += ("'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', " - "'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, " - "'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}") + message += ( + "'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', " + "'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, " + "'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}") token_count = count_tokens(message)["n_tokens"] response = make_openai_instruct_call(GPT_3_5_TURBO_INSTRUCT, message, token_count, @@ -690,5 +693,14 @@ def fetch_answer_tips(): return str(e) +@app.route('/grading_summary', methods=['POST']) +@jwt_required() +def grading_summary(): + try: + return calculate_grading_summary(request.get_json()) + except Exception as e: + return str(e) + + if __name__ == '__main__': app.run() diff --git a/module_grading_playground.py b/grading_summary/grading_summary.py similarity index 92% rename from module_grading_playground.py rename to grading_summary/grading_summary.py index 4f7c031..87d1e6c 100644 --- a/module_grading_playground.py +++ b/grading_summary/grading_summary.py @@ -3,7 +3,6 @@ import json import openai import os from dotenv import load_dotenv -from functools import reduce load_dotenv() openai.api_key = os.getenv("OPENAI_API_KEY") @@ -35,12 +34,7 @@ tools = [{ }] -# Input Format -# {'sections': Array of {'code': key, 'name': name, 'grade': grade}} -# Output -# {'sections': Array of {'code': key, 'name': name, 'grade': grade, 'evaluation': evaluation, 'suggestions': suggestions}} - -def grading_summary(body): +def calculate_grading_summary(body): extracted_sections = extract_existing_sections_from_body(body, section_keys) ret = [] From efef92343a5921b6890a6d21a4a379ae99e54bd8 Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Sat, 6 Jan 2024 19:01:31 +0000 Subject: [PATCH 3/6] comment --- app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app.py b/app.py index ac22564..a6897e3 100644 --- a/app.py +++ b/app.py @@ -696,6 +696,10 @@ def fetch_answer_tips(): @app.route('/grading_summary', methods=['POST']) @jwt_required() def grading_summary(): + # Body Format + # {'sections': Array of {'code': key, 'name': name, 'grade': grade}} + # Output Format + # {'sections': Array of {'code': key, 'name': name, 'grade': grade, 'evaluation': evaluation, 'suggestions': suggestions}} try: return calculate_grading_summary(request.get_json()) except Exception as e: From 046606a8ec4e03b25607993a2a71406b4a54a8c7 Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Sat, 6 Jan 2024 19:03:44 +0000 Subject: [PATCH 4/6] updated collection with new endpoint --- postman/ielts.postman_collection.json | 51 +++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/postman/ielts.postman_collection.json b/postman/ielts.postman_collection.json index e270b90..1a9bde3 100644 --- a/postman/ielts.postman_collection.json +++ b/postman/ielts.postman_collection.json @@ -1,9 +1,9 @@ { "info": { - "_postman_id": "1b901158-4228-426a-9c96-8cedc4df8470", + "_postman_id": "9905f8e4-f3b9-45e4-8ede-434c5de11eca", "name": "ielts", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "26107457" + "_exporter_id": "29491168" }, "item": [ { @@ -1104,6 +1104,53 @@ } }, "response": [] + }, + { + "name": "Fetch Answer Tips Copy", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{jwt_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"question\": \"When did Kendrick Lamar sign for TDE?\",\n \"answer\": \"Hello GPT.\",\n\t\t\"correct_answer\": \"2005\",\n \"context\": \"Kendrick Lamar Duckworth (born June 17, 1987) is an American rapper and songwriter. Known for his progressive musical styles and socially conscious songwriting, he is often considered one of the most influential hip hop artists of his generation. Born and raised in Compton, California, Lamar began his career as a teenager performing under the stage name K.Dot. He quickly garnered local attention which led to him signing a recording contract with Top Dawg Entertainment (TDE) in 2005.\"\n}\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://127.0.0.1:5000/fetch_tips", + "protocol": "http", + "host": [ + "127", + "0", + "0", + "1" + ], + "port": "5000", + "path": [ + "fetch_tips" + ] + } + }, + "response": [] } ] } \ No newline at end of file From 75df686cd1c5aa630e49248aa8484d6b1731992a Mon Sep 17 00:00:00 2001 From: Cristiano Ferreira Date: Sun, 7 Jan 2024 19:36:57 +0000 Subject: [PATCH 5/6] Refactored grading summary to fit previous existing files. --- app.py | 3 +- grading_summary/grading_summary.py | 88 --------------------------- helper/openai_interface.py | 82 +++++++++++++++++++++++++ postman/ielts.postman_collection.json | 2 +- 4 files changed, 84 insertions(+), 91 deletions(-) delete mode 100644 grading_summary/grading_summary.py diff --git a/app.py b/app.py index a6897e3..1eb1c57 100644 --- a/app.py +++ b/app.py @@ -10,8 +10,7 @@ from helper.firebase_helper import * from helper.heygen_api import create_videos_and_save_to_db from helper.speech_to_text_helper import * from helper.token_counter import count_tokens -from helper.openai_interface import make_openai_call, make_openai_instruct_call -from grading_summary.grading_summary import calculate_grading_summary +from helper.openai_interface import * import os import re import logging diff --git a/grading_summary/grading_summary.py b/grading_summary/grading_summary.py deleted file mode 100644 index 87d1e6c..0000000 --- a/grading_summary/grading_summary.py +++ /dev/null @@ -1,88 +0,0 @@ -import json - -import openai -import os -from dotenv import load_dotenv - -load_dotenv() -openai.api_key = os.getenv("OPENAI_API_KEY") - -chat_config = {'max_tokens': 1000, 'temperature': 0.2} -section_keys = ['reading', 'listening', 'writing', 'speaking', 'level'] -grade_top_limit = 9 - -tools = [{ - "type": "function", - "function": { - "name": "save_evaluation_and_suggestions", - "description": "Saves the evaluation and suggestions requested by input.", - "parameters": { - "type": "object", - "properties": { - "evaluation": { - "type": "string", - "description": "A comment on the IELTS section grade obtained in the specific section and what it could mean without suggestions.", - }, - "suggestions": { - "type": "string", - "description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.", - }, - }, - "required": ["evaluation", "suggestions"], - }, - } -}] - - -def calculate_grading_summary(body): - extracted_sections = extract_existing_sections_from_body(body, section_keys) - - ret = [] - - for section in extracted_sections: - openai_response_dict = calculate_section_grade_summary(section) - ret = ret + [{'code': section['code'], 'name': section['name'], 'grade': section['grade'], - 'evaluation': openai_response_dict['evaluation'], - 'suggestions': openai_response_dict['suggestions']}] - - return {'sections': ret} - - -def calculate_section_grade_summary(section): - res = openai.ChatCompletion.create( - model="gpt-3.5-turbo", - max_tokens=chat_config['max_tokens'], - temperature=chat_config['temperature'], - tools=tools, - messages=[ - { - "role": "user", - "content": "You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the grade obtained in the section. You should offer a comment on this grade with also suggestions on how to possibly get a better grade.", - }, - { - "role": "user", - "content": "Section: " + str(section['name']) + " Grade: " + str(section['grade']), - }, - {"role": "user", "content": "Speak in third person."}, - {"role": "user", "content": "Please save the evaluation and suggestions generated."} - ]) - - return parse_openai_response(res) - - -def parse_openai_response(response): - if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][ - 0] and 'tool_calls' in response['choices'][0]['message'] and isinstance( - response['choices'][0]['message']['tool_calls'], list) and len( - response['choices'][0]['message']['tool_calls']) > 0 and \ - response['choices'][0]['message']['tool_calls'][0]['function']['arguments']: - return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments']) - else: - return {'evaluation': "", 'suggestions': ""} - - -def extract_existing_sections_from_body(my_dict, keys_to_extract): - if 'sections' in my_dict and isinstance(my_dict['sections'], list) and len(my_dict['sections']) > 0: - return list(filter( - lambda item: 'code' in item and item['code'] in keys_to_extract and 'grade' in item and 'name' in item, - my_dict['sections'])) diff --git a/helper/openai_interface.py b/helper/openai_interface.py index 5c6bf99..ea8757d 100644 --- a/helper/openai_interface.py +++ b/helper/openai_interface.py @@ -16,6 +16,33 @@ TRY_LIMIT = 1 try_count = 0 +# GRADING SUMMARY +chat_config = {'max_tokens': 1000, 'temperature': 0.2} +section_keys = ['reading', 'listening', 'writing', 'speaking', 'level'] +grade_top_limit = 9 + +tools = [{ + "type": "function", + "function": { + "name": "save_evaluation_and_suggestions", + "description": "Saves the evaluation and suggestions requested by input.", + "parameters": { + "type": "object", + "properties": { + "evaluation": { + "type": "string", + "description": "A comment on the IELTS section grade obtained in the specific section and what it could mean without suggestions.", + }, + "suggestions": { + "type": "string", + "description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.", + }, + }, + "required": ["evaluation", "suggestions"], + }, + } +}] +### def process_response(input_string, quotation_check_field): if '{' in input_string: try: @@ -141,3 +168,58 @@ def make_openai_instruct_call(model, message: str, token_count, fields_to_check, else: try_count = 0 return processed_response + + +# GRADING SUMMARY +def calculate_grading_summary(body): + extracted_sections = extract_existing_sections_from_body(body, section_keys) + + ret = [] + + for section in extracted_sections: + openai_response_dict = calculate_section_grade_summary(section) + ret = ret + [{'code': section['code'], 'name': section['name'], 'grade': section['grade'], + 'evaluation': openai_response_dict['evaluation'], + 'suggestions': openai_response_dict['suggestions']}] + + return {'sections': ret} + + +def calculate_section_grade_summary(section): + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + max_tokens=chat_config['max_tokens'], + temperature=chat_config['temperature'], + tools=tools, + messages=[ + { + "role": "user", + "content": "You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the grade obtained in the section. You should offer a comment on this grade with also suggestions on how to possibly get a better grade.", + }, + { + "role": "user", + "content": "Section: " + str(section['name']) + " Grade: " + str(section['grade']), + }, + {"role": "user", "content": "Speak in third person."}, + {"role": "user", "content": "Please save the evaluation and suggestions generated."} + ]) + + return parse_openai_response(res) + + +def parse_openai_response(response): + if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][ + 0] and 'tool_calls' in response['choices'][0]['message'] and isinstance( + response['choices'][0]['message']['tool_calls'], list) and len( + response['choices'][0]['message']['tool_calls']) > 0 and \ + response['choices'][0]['message']['tool_calls'][0]['function']['arguments']: + return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments']) + else: + return {'evaluation': "", 'suggestions': ""} + + +def extract_existing_sections_from_body(my_dict, keys_to_extract): + if 'sections' in my_dict and isinstance(my_dict['sections'], list) and len(my_dict['sections']) > 0: + return list(filter( + lambda item: 'code' in item and item['code'] in keys_to_extract and 'grade' in item and 'name' in item, + my_dict['sections'])) \ No newline at end of file diff --git a/postman/ielts.postman_collection.json b/postman/ielts.postman_collection.json index 1a9bde3..78a4003 100644 --- a/postman/ielts.postman_collection.json +++ b/postman/ielts.postman_collection.json @@ -1106,7 +1106,7 @@ "response": [] }, { - "name": "Fetch Answer Tips Copy", + "name": "Get Grading Summary", "request": { "auth": { "type": "bearer", From e7a96c6880eb60320f8cb5270116408b5f082964 Mon Sep 17 00:00:00 2001 From: Cristiano Ferreira Date: Sun, 7 Jan 2024 19:38:24 +0000 Subject: [PATCH 6/6] Reformat. --- helper/openai_interface.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/helper/openai_interface.py b/helper/openai_interface.py index ea8757d..f5d9f39 100644 --- a/helper/openai_interface.py +++ b/helper/openai_interface.py @@ -42,7 +42,10 @@ tools = [{ }, } }] + + ### + def process_response(input_string, quotation_check_field): if '{' in input_string: try: @@ -71,6 +74,7 @@ def process_response(input_string, quotation_check_field): else: return input_string + def parse_string(to_parse: str): parsed_string = to_parse.replace("\"", "\\\"") pattern = r"(? 0: return list(filter( lambda item: 'code' in item and item['code'] in keys_to_extract and 'grade' in item and 'name' in item, - my_dict['sections'])) \ No newline at end of file + my_dict['sections']))