devops/encoach_backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged in grading-summary (pull request #7)

Browse Source 
Grading Summary Endpoint Logic

Approved-by: Tiago Ribeiro
Approved-by: Cristiano Ferreira
This commit is contained in:
Pedro Fonseca
2024-01-07 22:47:00 +00:00
committed by Tiago Ribeiro
parent 63823a01de e7a96c6880
commit 6baf669216
3 changed files with 168 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
app.py
View File

@@ -10,7 +10,7 @@ from helper.firebase_helper import *
from helper.heygen_api import create_videos_and_save_to_db
from helper.speech_to_text_helper import *
from helper.token_counter import count_tokens
from helper.openai_interface import make_openai_call, make_openai_instruct_call
from helper.openai_interface import *
import os
import re
import logging
@@ -37,6 +37,7 @@ thread_event = threading.Event()
logging.basicConfig(level=logging.DEBUG, # Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
format='%(asctime)s - %(levelname)s - %(message)s')
@app.route('/healthcheck', methods=['GET'])
def healthcheck():
return {"healthy": True}
@@ -511,7 +512,8 @@ def grade_speaking_task_3():
formatted_text += f"**Answer {i}:**\n{entry['answer']}\n\n"
message += formatted_text
message += ("'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', "
message += (
"'\n\nProvide your answer on the following json format: {'comment': 'comment about answer quality', "
"'overall': 0.0, 'task_response': {'Fluency and Coherence': 0.0, 'Lexical Resource': 0.0, "
"'Grammatical Range and Accuracy': 0.0, 'Pronunciation': 0.0}}")
@@ -690,5 +692,18 @@ def fetch_answer_tips():
return str(e)
@app.route('/grading_summary', methods=['POST'])
@jwt_required()
def grading_summary():
# Body Format
# {'sections': Array of {'code': key, 'name': name, 'grade': grade}}
# Output Format
# {'sections': Array of {'code': key, 'name': name, 'grade': grade, 'evaluation': evaluation, 'suggestions': suggestions}}
try:
return calculate_grading_summary(request.get_json())
except Exception as e:
return str(e)
if __name__ == '__main__':
app.run()

88
helper/openai_interface.py
View File

@@ -16,6 +16,36 @@ TRY_LIMIT = 1
try_count = 0
# GRADING SUMMARY
chat_config = {'max_tokens': 1000, 'temperature': 0.2}
section_keys = ['reading', 'listening', 'writing', 'speaking', 'level']
grade_top_limit = 9
tools = [{
"type": "function",
"function": {
"name": "save_evaluation_and_suggestions",
"description": "Saves the evaluation and suggestions requested by input.",
"parameters": {
"type": "object",
"properties": {
"evaluation": {
"type": "string",
"description": "A comment on the IELTS section grade obtained in the specific section and what it could mean without suggestions.",
},
"suggestions": {
"type": "string",
"description": "A small paragraph text with suggestions on how to possibly get a better grade than the one obtained.",
},
},
"required": ["evaluation", "suggestions"],
},
}
}]
###
def process_response(input_string, quotation_check_field):
if '{' in input_string:
try:
@@ -44,6 +74,7 @@ def process_response(input_string, quotation_check_field):
else:
return input_string
def parse_string(to_parse: str):
parsed_string = to_parse.replace("\"", "\\\"")
pattern = r"(?<!\w)'|'(?!\w)"
@@ -80,6 +111,7 @@ def parse_string_2(to_parse: str):
to_parse = to_parse.replace(":", ": ")
return to_parse
def remove_special_chars_and_escapes(input_string):
parsed_string = input_string.replace("\\\"", "'")
parsed_string = parsed_string.replace("\n\n", " ")
@@ -122,6 +154,7 @@ def make_openai_call(model, messages, token_count, fields_to_check, temperature)
try_count = 0
return processed_response
def make_openai_instruct_call(model, message: str, token_count, fields_to_check, temperature):
global try_count
response = openai.Completion.create(
@@ -141,3 +174,58 @@ def make_openai_instruct_call(model, message: str, token_count, fields_to_check,
else:
try_count = 0
return processed_response
# GRADING SUMMARY
def calculate_grading_summary(body):
extracted_sections = extract_existing_sections_from_body(body, section_keys)
ret = []
for section in extracted_sections:
openai_response_dict = calculate_section_grade_summary(section)
ret = ret + [{'code': section['code'], 'name': section['name'], 'grade': section['grade'],
'evaluation': openai_response_dict['evaluation'],
'suggestions': openai_response_dict['suggestions']}]
return {'sections': ret}
def calculate_section_grade_summary(section):
res = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
max_tokens=chat_config['max_tokens'],
temperature=chat_config['temperature'],
tools=tools,
messages=[
{
"role": "user",
"content": "You are a IELTS test section grade evaluator. You will receive a IELTS test section name and the grade obtained in the section. You should offer a comment on this grade with also suggestions on how to possibly get a better grade.",
},
{
"role": "user",
"content": "Section: " + str(section['name']) + " Grade: " + str(section['grade']),
},
{"role": "user", "content": "Speak in third person."},
{"role": "user", "content": "Please save the evaluation and suggestions generated."}
])
return parse_openai_response(res)
def parse_openai_response(response):
if 'choices' in response and len(response['choices']) > 0 and 'message' in response['choices'][
0] and 'tool_calls' in response['choices'][0]['message'] and isinstance(
response['choices'][0]['message']['tool_calls'], list) and len(
response['choices'][0]['message']['tool_calls']) > 0 and \
response['choices'][0]['message']['tool_calls'][0]['function']['arguments']:
return json.loads(response['choices'][0]['message']['tool_calls'][0]['function']['arguments'])
else:
return {'evaluation': "", 'suggestions': ""}
def extract_existing_sections_from_body(my_dict, keys_to_extract):
if 'sections' in my_dict and isinstance(my_dict['sections'], list) and len(my_dict['sections']) > 0:
return list(filter(
lambda item: 'code' in item and item['code'] in keys_to_extract and 'grade' in item and 'name' in item,
my_dict['sections']))

51
postman/ielts.postman_collection.json
View File

@@ -1,9 +1,9 @@
{
"info": {
"_postman_id": "1b901158-4228-426a-9c96-8cedc4df8470",
"_postman_id": "9905f8e4-f3b9-45e4-8ede-434c5de11eca",
"name": "ielts",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
"_exporter_id": "26107457"
"_exporter_id": "29491168"
},
"item": [
{
@@ -1104,6 +1104,53 @@
}
},
"response": []
},
{
"name": "Get Grading Summary",
"request": {
"auth": {
"type": "bearer",
"bearer": [
{
"key": "token",
"value": "{{jwt_token}}",
"type": "string"
}
]
},
"method": "POST",
"header": [
{
"key": "Content-Type",
"value": "application/json",
"type": "text"
}
],
"body": {
"mode": "raw",
"raw": "{\n \"question\": \"When did Kendrick Lamar sign for TDE?\",\n \"answer\": \"Hello GPT.\",\n\t\t\"correct_answer\": \"2005\",\n \"context\": \"Kendrick Lamar Duckworth (born June 17, 1987) is an American rapper and songwriter. Known for his progressive musical styles and socially conscious songwriting, he is often considered one of the most influential hip hop artists of his generation. Born and raised in Compton, California, Lamar began his career as a teenager performing under the stage name K.Dot. He quickly garnered local attention which led to him signing a recording contract with Top Dawg Entertainment (TDE) in 2005.\"\n}\n",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "http://127.0.0.1:5000/fetch_tips",
"protocol": "http",
"host": [
"127",
"0",
"0",
"1"
],
"port": "5000",
"path": [
"fetch_tips"
]
}
},
"response": []
}
]
}