From 4e1ad6dc67777c834bb5243515d44fd33e768add Mon Sep 17 00:00:00 2001 From: Cristiano Ferreira Date: Fri, 23 Jun 2023 00:05:48 +0100 Subject: [PATCH] Add speaking endpoints and clean code. --- .env | 3 +- app.py | 141 +++++++++--------- firebase-configs/mti-ielts-626a2dcf6091.json | 13 ++ helper/api_messages.py | 143 +++++++++++++++++++ helper/firebase_helper.py | 15 ++ helper/openai_interface.py | 29 ++-- helper/speech_to_text_helper.py | 11 ++ requirements.txt | Bin 262 -> 348 bytes sp1_playground.py | 8 +- 9 files changed, 274 insertions(+), 89 deletions(-) create mode 100644 firebase-configs/mti-ielts-626a2dcf6091.json create mode 100644 helper/api_messages.py create mode 100644 helper/firebase_helper.py create mode 100644 helper/speech_to_text_helper.py diff --git a/.env b/.env index 49943ef..b9243a1 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ OPENAI_API_KEY=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 JWT_SECRET_KEY=6e9c124ba92e8814719dcb0f21200c8aa4d0f119a994ac5e06eb90a366c83ab2 -JWT_TEST_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0In0.Emrs2D3BmMP4b3zMjw0fJTPeyMwWEBDbxx2vvaWguO0 \ No newline at end of file +JWT_TEST_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0In0.Emrs2D3BmMP4b3zMjw0fJTPeyMwWEBDbxx2vvaWguO0 +GOOGLE_APPLICATION_CREDENTIALS=firebase-configs/mti-ielts-626a2dcf6091.json \ No newline at end of file diff --git a/app.py b/app.py index c24a135..1050121 100644 --- a/app.py +++ b/app.py @@ -1,9 +1,15 @@ from flask import Flask, request from flask_jwt_extended import JWTManager, jwt_required from functools import reduce +import firebase_admin +from firebase_admin import credentials +from helper.api_messages import QuestionType, get_grading_messages, get_question_gen_messages +from helper.firebase_helper import download_firebase_file +from helper.speech_to_text_helper import speech_to_text from helper.token_counter import count_tokens from helper.openai_interface import make_openai_call import os +import uuid from dotenv import load_dotenv @@ -14,91 +20,80 @@ app = Flask(__name__) app.config['JWT_SECRET_KEY'] = os.getenv("JWT_SECRET_KEY") jwt = JWTManager(app) +# Initialize Firebase Admin SDK +cred = credentials.Certificate(os.getenv("GOOGLE_APPLICATION_CREDENTIALS")) +firebase_admin.initialize_app(cred) + GRADING_TEMPERATURE = 0.1 GEN_QUESTION_TEMPERATURE = 0.7 WRITING_TASK_2_POST_FIELDS = ['overall', 'comment', 'task_response'] WRITING_TASK_2_GET_FIELDS = ['question'] +FIREBASE_BUCKET = 'mti-ielts.appspot.com' +AUDIO_FILES_PATH = 'download-audio/' + + @app.route('/writing_task2', methods=['POST']) @jwt_required() def grade_writing_task(): - data = request.get_json() # Assuming the request data is in JSON format - question = data.get('question') - answer = data.get('answer') - messages = [ - { - "role": "system", - "content": "You are a IELTS examiner.", - }, - { - "role": "system", - "content": f"The question you have to grade is of type Writing Task 2 and is the following: {question}", - }, - { - "role": "user", - "content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, " - "in the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, 'task_response': {'Task Achievement': 8.0, " - "'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, 'Grammatical Range and Accuracy': " - "6.0}}", - }, - { - "role": "user", - "content": "Example output: { 'comment': 'Overall, the response is good but there are some areas that need " - "improvement.\n\nIn terms of Task Achievement, the writer has addressed all parts of the question " - "and has provided a clear opinion on the topic. However, some of the points made are not fully " - "developed or supported with examples.\n\nIn terms of Coherence and Cohesion, there is a clear " - "structure to the response with an introduction, body paragraphs and conclusion. However, there " - "are some issues with cohesion as some sentences do not flow smoothly from one to another.\n\nIn " - "terms of Lexical Resource, there is a good range of vocabulary used throughout the response and " - "some less common words have been used effectively.\n\nIn terms of Grammatical Range and Accuracy, " - "there are some errors in grammar and sentence structure which affect clarity in places.\n\nOverall, " - "this response would score a band 6.5.', 'overall': 6.5, 'task_response': " - "{ 'Coherence and Cohesion': 6.5, 'Grammatical Range and Accuracy': 6.0, 'Lexical Resource': 7.0, " - "'Task Achievement': 7.0}}", - }, - { - "role": "user", - "content": f"Evaluate this answer according to ielts grading system: {answer}", - }, - ] - token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], - map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) - response = make_openai_call(messages, token_count, WRITING_TASK_2_POST_FIELDS, GRADING_TEMPERATURE) - return response + try: + data = request.get_json() + question = data.get('question') + answer = data.get('answer') + messages = get_grading_messages(QuestionType.WRITING_TASK_2, question, answer) + token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], + map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) + response = make_openai_call(messages, token_count, WRITING_TASK_2_POST_FIELDS, GRADING_TEMPERATURE) + return response + except Exception as e: + return str(e) + @app.route('/writing_task2', methods=['GET']) @jwt_required() def get_writing_task_question(): - messages = [ - { - "role": "system", - "content": "You are a IELTS program that generates questions for the exams.", - }, - { - "role": "system", - "content": "The question you have to generate is of type Writing Task 2 and is the following.", - }, - { - "role": "user", - "content": "It is mandatory for you to provide your response with the question " - "in the following json format: {'question': 'question'}", - }, - { - "role": "user", - "content": "Example output: { 'question': 'We are becoming increasingly dependent on computers. " - "They are used in businesses, hospitals, crime detection and even to fly planes. What things will " - "they be used for in the future? Is this dependence on computers a good thing or should we he more " - "auspicious of their benefits?'}", - }, - { - "role": "user", - "content": "Generate a question for IELTS exam Writing Task 2.", - }, - ] - token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], - map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) - response = make_openai_call(messages, token_count, WRITING_TASK_2_GET_FIELDS, GEN_QUESTION_TEMPERATURE) - return response + try: + messages = get_question_gen_messages(QuestionType.WRITING_TASK_2) + token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], + map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) + response = make_openai_call(messages, token_count, WRITING_TASK_2_GET_FIELDS, GEN_QUESTION_TEMPERATURE) + return response + except Exception as e: + return str(e) + +@app.route('/speaking_task', methods=['POST']) +@jwt_required() +def grade_speaking_task(): + sound_file_name = AUDIO_FILES_PATH + str(uuid.uuid4()) + try: + data = request.get_json() + question = data.get('question') + answer_firebase_path = data.get('answer') + + download_firebase_file(FIREBASE_BUCKET, answer_firebase_path, sound_file_name) + answer = speech_to_text(sound_file_name) + + messages = get_grading_messages(QuestionType.SPEAKING, question, answer) + token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], + map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) + response = make_openai_call(messages, token_count, WRITING_TASK_2_POST_FIELDS, GRADING_TEMPERATURE) + os.remove(sound_file_name) + return response + except Exception as e: + os.remove(sound_file_name) + return str(e) + +@app.route('/speaking_task', methods=['GET']) +@jwt_required() +def get_speaking_task_question(): + try: + messages = get_question_gen_messages(QuestionType.SPEAKING) + token_count = reduce(lambda count, item: count + count_tokens(item)['n_tokens'], + map(lambda x: x["content"], filter(lambda x: "content" in x, messages)), 0) + response = make_openai_call(messages, token_count, WRITING_TASK_2_GET_FIELDS, GEN_QUESTION_TEMPERATURE) + return response + except Exception as e: + return str(e) if __name__ == '__main__': diff --git a/firebase-configs/mti-ielts-626a2dcf6091.json b/firebase-configs/mti-ielts-626a2dcf6091.json new file mode 100644 index 0000000..3bf3594 --- /dev/null +++ b/firebase-configs/mti-ielts-626a2dcf6091.json @@ -0,0 +1,13 @@ +{ + "type": "service_account", + "project_id": "mti-ielts", + "private_key_id": "626a2dcf60916a1b5011f388495b8f9c4fc065ef", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDuaLgLNa5yb5LI\nPZYa7qav0URgCF7miK3dUXIBoABQ+U6y1LwdsIiJqHZ4Cm2lotTqeTGOIV83PuA6\n9H/TwnvsHH8jilmsPxO5OX7AyZSDPvN45nJrgQ21RKZCYQGVetBMGhclCRbYFraS\nE6X/p6gSOpSqZ5fLz8BbdCMfib6HSfDmBkYTK42X6d2eNNwLM1wLbE8RmCGwRATC\nQFfMhjlvQcSJ1EDMfkMUUE9U/ux77wfHqs1d+7utVcQTIMFAP9fo1ynJlwp8D1HQ\ntalB6kkpuDQetUR0A1FHMMJekhmuRDUMfokX1F9JfUjR0OetuD3KEH5y2asxC2+0\n8JYcwbvlAgMBAAECggEAKaaW3LJ8rxZp/NyxkDP4YAf9248q0Ti4s00qzzjeRUdA\n5gI/eSphuDb7t34O6NyZOPuCWlPfOB4ee35CpMK59qaF2bYuc2azseznBZRSA1no\nnEsaW0i5Fd2P9FHRPoWtxVXbjEdZu9e//qY7Hn5yYPjmBx1BCkTZ1MBl8HkWlbjR\nbu18uveg5Vg6Wc+rnPmH/gMRLLpq9iQBpzXWT8Mj+k48O8GnW6v8S3R027ymqUou\n3W5b69xDGn0nwxgLIVzdxjoo7RnpjD3mP0x4faiBhScVgFhwZP8hqBeVyqbV5dMh\nfF+p9zLOeilFLJEjH1lZbZAb8wwP23LozIXJWFG3oQKBgQD6COCJ7hNSx9/AzDhO\nh73hKH/KSOJtxHc8795hcZjy9HJkoM45Fm7o2QGZzsZmV+N6VU0BjoDQAyftCq+G\ndIX0wcAGJIsLuQ9K00WI2hn7Uq1gjUl0d9XEorogKa1ZNTLL/9By/xnA7sEpI6Ng\nIsKQ4R2CfqNFU4bs1nyKWCWudQKBgQD0GNYwZt3xV2YBATVYsrvg1OGO/tmkCJ8Y\nLOdM0L+8WMCgw0uQcNFF9uqq6/oFgq7tOvpeZDsY8onRy55saaMT+Lr4xs0sj5B0\ns5Hqc0L37tdXXXXEne8WABMBF9injNgNbAm9W0kqME2Stc53OJQPj2DBdYxWSr8v\n36imCwoJsQKBgH0BBSlQQo7naKFeOGRijvbLpZ//clzIlYh8r+Rtw7brqWlPz+pQ\noeB95cP80coG9K6LiPVXRmU4vrRO3FRPW01ztEod6PpSaifRmnkB+W1h91ZHLMsy\nwkgNxxofXBA2fY/p9FAZ48lGVIH51EtS9Y0zTuqX347gZJtx3E/aI/SlAoGBAJer\nCwM+F2+K352GM7BuNiDoBVLFdVPf64Ko+/sVxdzwxJffYQdZoh634m3bfBmKbsiG\nmeSmoLXKlenefAxewu544SwM0pV6isaIgQTNI3JMXE8ziiZl/5WK7EQEniDVebU1\nSQP4QYjORJUBFE2twQm+C9+I+27uuMa1UOQC/fSxAoGBANuWloacqGfws6nbHvqF\nLZKlkKNPI/0sC+6VlqjoHn5LQz3lcFM1+iKSQIGJvJyru2ODgv2Lmq2W+cx+HMeq\n0BSetK4XtalmO9YflH7uMgvOEVewf4uJ2d+4I1pbY9aI1gHaZ1EUiiy6Ds4kAK8s\nTQqp88pfTbOnkdJBVi0AWs5B\n-----END PRIVATE KEY-----\n", + "client_email": "firebase-adminsdk-dyg6p@mti-ielts.iam.gserviceaccount.com", + "client_id": "104980563453519094431", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-dyg6p%40mti-ielts.iam.gserviceaccount.com", + "universe_domain": "googleapis.com" +} diff --git a/helper/api_messages.py b/helper/api_messages.py new file mode 100644 index 0000000..1e5f3a6 --- /dev/null +++ b/helper/api_messages.py @@ -0,0 +1,143 @@ +from enum import Enum + + +class QuestionType(Enum): + WRITING_TASK_2 = "Writing Task 2" + SPEAKING = "Speaking Task" + + +def get_grading_messages(question_type: QuestionType, question: str, answer: str): + if QuestionType.WRITING_TASK_2 == question_type: + return [ + { + "role": "system", + "content": "You are a IELTS examiner.", + }, + { + "role": "system", + "content": f"The question you have to grade is of type Writing Task 2 and is the following: {question}", + }, + { + "role": "user", + "content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, " + "with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, " + "'task_response': {'Task Achievement': 8.0, 'Coherence and Cohesion': 6.5, 'Lexical Resource': 7.5, " + "'Grammatical Range and Accuracy': 6.0}}", + }, + { + "role": "user", + "content": "Example output: { 'comment': 'Overall, the response is good but there are some areas that need " + "improvement.\n\nIn terms of Task Achievement, the writer has addressed all parts of the question " + "and has provided a clear opinion on the topic. However, some of the points made are not fully " + "developed or supported with examples.\n\nIn terms of Coherence and Cohesion, there is a clear " + "structure to the response with an introduction, body paragraphs and conclusion. However, there " + "are some issues with cohesion as some sentences do not flow smoothly from one to another.\n\nIn " + "terms of Lexical Resource, there is a good range of vocabulary used throughout the response and " + "some less common words have been used effectively.\n\nIn terms of Grammatical Range and Accuracy, " + "there are some errors in grammar and sentence structure which affect clarity in places.\n\nOverall, " + "this response would score a band 6.5.', 'overall': 6.5, 'task_response': " + "{ 'Coherence and Cohesion': 6.5, 'Grammatical Range and Accuracy': 6.0, 'Lexical Resource': 7.0, " + "'Task Achievement': 7.0}}", + }, + { + "role": "user", + "content": f"Evaluate this answer according to ielts grading system: {answer}", + }, + ] + elif QuestionType.SPEAKING == question_type: + return [ + { + "role": "user", + "content": "You are a IELTS examiner.", + }, + { + "role": "user", + "content": f"The question you have to grade is of type Speaking and is the following: {question}", + }, + { + "role": "user", + "content": "It is mandatory for you to provide your response with the overall grade and breakdown grades, " + "with just the following json format: {'comment': 'comment about answer quality', 'overall': 7.0, " + "'task_response': {'Fluency and Coherence': 8.0, 'Lexical Resource': 6.5, " + "'Grammatical Range and Accuracy': 7.5, 'Pronunciation': 6.0}}", + }, + { + "role": "user", + "content": "Example output: { 'comment': 'The candidate has provided a clear response to the question and has " + "given examples of how they spend their weekends. However, there are some issues with grammar and " + "pronunciation that affect the overall score. In terms of fluency and coherence, the candidate speaks " + "clearly and smoothly with only minor hesitations. They have also provided a well-organized response " + "that is easy to follow. Regarding lexical resource, the candidate has used a range of vocabulary " + "related to weekend activities but there are some errors in word choice that affect the meaning of " + "their sentences. In terms of grammatical range and accuracy, the candidate has used a mix of simple " + "and complex sentence structures but there are some errors in subject-verb agreement and preposition " + "use. Finally, regarding pronunciation, the candidate's speech is generally clear but there are some " + "issues with stress and intonation that make it difficult to understand at times.', 'overall': 6.5, " + "'task_response': {'Fluency and Coherence': 7.0, 'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.0," + " 'Pronunciation': 6.0}}", + }, + { + "role": "user", + "content": f"Evaluate this answer according to ielts grading system: {answer}", + }, + ] + else: + raise Exception("Question type not implemented: " + question_type.value) + + +def get_question_gen_messages(question_type: QuestionType): + if QuestionType.WRITING_TASK_2 == question_type: + return [ + { + "role": "system", + "content": "You are a IELTS program that generates questions for the exams.", + }, + { + "role": "system", + "content": "The question you have to generate is of type Writing Task 2.", + }, + { + "role": "user", + "content": "It is mandatory for you to provide your response with the question " + "just with the following json format: {'question': 'question'}", + }, + { + "role": "user", + "content": "Example output: { 'question': 'We are becoming increasingly dependent on computers. " + "They are used in businesses, hospitals, crime detection and even to fly planes. What things will " + "they be used for in the future? Is this dependence on computers a good thing or should we he more " + "auspicious of their benefits?'}", + }, + { + "role": "user", + "content": "Generate a question for IELTS exam Writing Task 2.", + }, + ] + elif QuestionType.SPEAKING == question_type: + return [ + { + "role": "system", + "content": "You are a IELTS program that generates questions for the exams.", + }, + { + "role": "system", + "content": "The question you have to generate is of type Speaking Task.", + }, + { + "role": "user", + "content": "It is mandatory for you to provide your response with the question " + "just with the following json format: {'question': 'question'}", + }, + { + "role": "user", + "content": "Example output: { 'question': 'Describe someone you know who does something well. You should say " + "who this person is, how do you know this person, what they do well and explain why you think this " + "person is so good at doing this.'}", + }, + { + "role": "user", + "content": "Generate a question for IELTS exam Speaking Task.", + }, + ] + else: + raise Exception("Question type not implemented: " + question_type.value) diff --git a/helper/firebase_helper.py b/helper/firebase_helper.py new file mode 100644 index 0000000..7023379 --- /dev/null +++ b/helper/firebase_helper.py @@ -0,0 +1,15 @@ +from google.cloud import storage + +def download_firebase_file(bucket_name, source_blob_name, destination_file_name): + # Downloads a file from Firebase Storage. + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name) + print(f"File downloaded to {destination_file_name}") + return destination_file_name + + + + + diff --git a/helper/openai_interface.py b/helper/openai_interface.py index 8fa8a08..0b825e6 100644 --- a/helper/openai_interface.py +++ b/helper/openai_interface.py @@ -1,6 +1,7 @@ import json import openai import os +import re from dotenv import load_dotenv @@ -16,15 +17,25 @@ TRY_LIMIT = 1 try_count = 0 def process_response(input_string): - json_obj = {} - parsed_string = input_string.replace("'", "\"") - parsed_string = parsed_string.replace("\n\n", " ") - try: - json_obj = json.loads(parsed_string) - except json.JSONDecodeError: - print("Invalid JSON string!") + if '{' in input_string: + try: + # Find the index of the first occurrence of '{' + index = input_string.index('{') + # Extract everything after the first '{' (inclusive) + result = input_string[index:] - return json_obj + parsed_string = result.replace("\"", "\\\"") + pattern = r"(?= TRY_LIMIT: try_count = 0 return result["choices"][0]["message"]["content"] diff --git a/helper/speech_to_text_helper.py b/helper/speech_to_text_helper.py new file mode 100644 index 0000000..0e8598a --- /dev/null +++ b/helper/speech_to_text_helper.py @@ -0,0 +1,11 @@ +import whisper +import os + +def speech_to_text(file_path): + if os.path.exists(file_path): + model = whisper.load_model("base") + result = model.transcribe(file_path, fp16=False, language='English', verbose=False) + return result["text"] + else: + print("File not found:", file_path) + raise Exception("File " + file_path + " not found.") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7bf8d0a3b5bb88f9705972167becabcb83caf435..210df3e4fd3c97dcde78dc11d84b5d4adc04d14a 100644 GIT binary patch delta 95 zcmZo;y2CW#RcJaxJ`koe3oJ#h7=&Xn4tv7E@DUo%BC{#GH@}ZF=PVy jsSHU#v0||LM4-xCplBXL9fK`{8G{~!ArKo(d?^b63k(zN delta 10 Rcmcb^)W$U7)no@o82}g#1JM8g diff --git a/sp1_playground.py b/sp1_playground.py index 3f16d70..1c6d834 100644 --- a/sp1_playground.py +++ b/sp1_playground.py @@ -38,18 +38,14 @@ def correct_answer( "content": f"The question you have to grade is of type {question_type} and is the following: {question}", }, { - "role": "system", + "role": "user", "content": "Please provide a JSON object response with the overall grade and breakdown grades, " "formatted as follows: {'overall': 7.0, 'task_response': {'Fluency and Coherence': 8.0, " "'Lexical Resource': 6.5, 'Grammatical Range and Accuracy': 7.5, 'Pronunciation': " "6.0}}", }, { - "role": "system", - "content": "Don't give explanations for the grades, just provide the json with the grades.", - }, - { - "role": "system", + "role": "user", "content": "If the answer is unrelated to the question give it the minimum grade.", }, {