Async release

app/services/impl/third_parties/__init__.py

@@ -0,0 +1,13 @@
+from .aws_polly import AWSPolly
+from .heygen import Heygen
+from .openai import OpenAI
+from .whisper import OpenAIWhisper
+from .gpt_zero import GPTZero
+
+__all__ = [
+    "AWSPolly",
+    "Heygen",
+    "OpenAI",
+    "OpenAIWhisper",
+    "GPTZero"
+]

app/services/impl/third_parties/aws_polly.py

@@ -0,0 +1,87 @@
+import random
+from typing import Union
+
+import aiofiles
+from aiobotocore.client import BaseClient
+
+from app.services.abc import ITextToSpeechService
+from app.configs.constants import NeuralVoices
+
+
+class AWSPolly(ITextToSpeechService):
+
+    def __init__(self, client: BaseClient):
+        self._client = client
+
+    async def synthesize_speech(self, text: str, voice: str, engine: str = "neural", output_format: str = "mp3"):
+        tts_response = await self._client.synthesize_speech(
+            Engine=engine,
+            Text=text,
+            OutputFormat=output_format,
+            VoiceId=voice
+        )
+        return await tts_response['AudioStream'].read()
+
+    async def text_to_speech(self, text: Union[list[str], str], file_name: str):
+        if isinstance(text, str):
+            audio_segments = await self._text_to_speech(text)
+        elif isinstance(text, list):
+            audio_segments = await self._conversation_to_speech(text)
+        else:
+            raise ValueError("Unsupported argument for text_to_speech")
+
+        final_message = await self.synthesize_speech(
+            "This audio recording, for the listening exercise, has finished.",
+            "Stephen"
+        )
+
+        # Add finish message
+        audio_segments.append(final_message)
+
+        # Combine the audio segments into a single audio file
+        combined_audio = b"".join(audio_segments)
+        # Save the combined audio to a single file
+        async with aiofiles.open(file_name, "wb") as f:
+            await f.write(combined_audio)
+
+        print("Speech segments saved to " + file_name)
+
+    async def _text_to_speech(self, text: str):
+        voice = random.choice(NeuralVoices.ALL_NEURAL_VOICES)['Id']
+        # Initialize an empty list to store audio segments
+        audio_segments = []
+        for part in self._divide_text(text):
+            audio_segments.append(await self.synthesize_speech(part, voice))
+
+        return audio_segments
+
+    async def _conversation_to_speech(self, conversation: list):
+        # Initialize an empty list to store audio segments
+        audio_segments = []
+        # Iterate through the text segments, convert to audio segments, and store them
+        for segment in conversation:
+            audio_segments.append(await self.synthesize_speech(segment["text"], segment["voice"]))
+
+        return audio_segments
+
+    @staticmethod
+    def _divide_text(text, max_length=3000):
+        if len(text) <= max_length:
+            return [text]
+
+        divisions = []
+        current_position = 0
+
+        while current_position < len(text):
+            next_position = min(current_position + max_length, len(text))
+            next_period_position = text.rfind('.', current_position, next_position)
+
+            if next_period_position != -1 and next_period_position > current_position:
+                divisions.append(text[current_position:next_period_position + 1])
+                current_position = next_period_position + 1
+            else:
+                # If no '.' found in the next chunk, split at max_length
+                divisions.append(text[current_position:next_position])
+                current_position = next_position
+
+        return divisions

app/services/impl/third_parties/gpt_zero.py

@@ -0,0 +1,52 @@
+from logging import getLogger
+from typing import Dict, Optional
+
+from httpx import AsyncClient
+
+from app.services.abc.third_parties.ai_detector import IAIDetectorService
+
+
+class GPTZero(IAIDetectorService):
+
+    _GPT_ZERO_ENDPOINT = 'https://api.gptzero.me/v2/predict/text'
+
+    def __init__(self, client: AsyncClient, gpt_zero_key: str):
+        self._header = {
+            'x-api-key': gpt_zero_key
+        }
+        self._http_client = client
+        self._logger = getLogger(__name__)
+
+    async def run_detection(self, text: str):
+        data = {
+            'document': text,
+            'version': '',
+            'multilingual': False
+        }
+
+        response = await self._http_client.post(self._GPT_ZERO_ENDPOINT, headers=self._header, json=data)
+        if response.status_code != 200:
+            return None
+        return self._parse_detection(response.json())
+
+    def _parse_detection(self, response: Dict) -> Optional[Dict]:
+        try:
+            text_scan = response["documents"][0]
+
+            filtered_sentences = [
+                {
+                    "sentence": item["sentence"],
+                    "highlight_sentence_for_ai": item["highlight_sentence_for_ai"]
+                }
+                for item in text_scan["sentences"]
+            ]
+
+            return {
+                "class_probabilities": text_scan["class_probabilities"],
+                "confidence_category": text_scan["confidence_category"],
+                "predicted_class": text_scan["predicted_class"],
+                "sentences": filtered_sentences
+            }
+        except Exception as e:
+            self._logger.error(f'Failed to parse GPT\'s Zero response: {str(e)}')
+            return None

app/services/impl/third_parties/heygen.py

@@ -0,0 +1,90 @@
+import asyncio
+import os
+import logging
+import aiofiles
+
+from httpx import AsyncClient
+
+from app.services.abc import IVideoGeneratorService
+
+
+class Heygen(IVideoGeneratorService):
+
+    # TODO: Not used, remove if not necessary
+    # CREATE_VIDEO_URL = 'https://api.heygen.com/v1/template.generate'
+
+    _GET_VIDEO_URL = 'https://api.heygen.com/v1/video_status.get'
+
+    def __init__(self, client: AsyncClient, heygen_token: str):
+        self._get_header = {
+            'X-Api-Key': heygen_token
+        }
+        self._post_header = {
+            'X-Api-Key': heygen_token,
+            'Content-Type': 'application/json'
+        }
+        self._http_client = client
+        self._logger = logging.getLogger(__name__)
+
+    async def create_video(self, text: str, avatar: str):
+        # POST TO CREATE VIDEO
+        create_video_url = 'https://api.heygen.com/v2/template/' + avatar + '/generate'
+        data = {
+            "test": False,
+            "caption": False,
+            "title": "video_title",
+            "variables": {
+                "script_here": {
+                    "name": "script_here",
+                    "type": "text",
+                    "properties": {
+                        "content": text
+                    }
+                }
+            }
+        }
+        response = await self._http_client.post(create_video_url, headers=self._post_header, json=data)
+        self._logger.info(response.status_code)
+        self._logger.info(response.json())
+
+        # GET TO CHECK STATUS AND GET VIDEO WHEN READY
+        video_id = response.json()["data"]["video_id"]
+        params = {
+            'video_id': response.json()["data"]["video_id"]
+        }
+        response = {}
+        status = "processing"
+        error = None
+
+        while status != "completed" and error is None:
+            response = await self._http_client.get(self._GET_VIDEO_URL, headers=self._get_header, params=params)
+            response_data = response.json()
+
+            status = response_data["data"]["status"]
+            error = response_data["data"]["error"]
+
+            if status != "completed" and error is None:
+                self._logger.info(f"Status: {status}")
+                await asyncio.sleep(10)  # Wait for 10 second before the next request
+
+        self._logger.info(response.status_code)
+        self._logger.info(response.json())
+
+        # DOWNLOAD VIDEO
+        download_url = response.json()['data']['video_url']
+        output_directory = 'download-video/'
+        output_filename = video_id + '.mp4'
+
+        response = await self._http_client.get(download_url)
+
+        if response.status_code == 200:
+            os.makedirs(output_directory, exist_ok=True)  # Create the directory if it doesn't exist
+            output_path = os.path.join(output_directory, output_filename)
+            async with aiofiles.open(output_path, 'wb') as f:
+                await f.write(response.content)
+            self._logger.info(f"File '{output_filename}' downloaded successfully.")
+            return output_filename
+        else:
+            self._logger.error(f"Failed to download file. Status code: {response.status_code}")
+            return None
+

app/services/impl/third_parties/openai.py

@@ -0,0 +1,97 @@
+import json
+import re
+import logging
+from typing import List, Optional
+from openai import AsyncOpenAI
+from openai.types.chat import ChatCompletionMessageParam
+
+from app.services.abc import ILLMService
+from app.helpers import count_tokens
+from app.configs.constants import BLACKLISTED_WORDS
+
+
+class OpenAI(ILLMService):
+
+    MAX_TOKENS = 4097
+    TRY_LIMIT = 2
+
+    def __init__(self, client: AsyncOpenAI):
+        self._client = client
+        self._logger = logging.getLogger(__name__)
+
+    async def prediction(
+            self,
+            model: str,
+            messages: List[ChatCompletionMessageParam],
+            fields_to_check: Optional[List[str]],
+            temperature: float,
+            check_blacklisted: bool = True,
+            token_count: int = -1
+    ):
+        if token_count == -1:
+            token_count = self._count_total_tokens(messages)
+        return await self._prediction(model, messages, token_count, fields_to_check, temperature, 0, check_blacklisted)
+
+    async def _prediction(
+            self,
+            model: str,
+            messages: List[ChatCompletionMessageParam],
+            token_count: int,
+            fields_to_check: Optional[List[str]],
+            temperature: float,
+            try_count: int,
+            check_blacklisted: bool,
+    ):
+        result = await self._client.chat.completions.create(
+            model=model,
+            max_tokens=int(self.MAX_TOKENS - token_count - 300),
+            temperature=float(temperature),
+            messages=messages,
+            response_format={"type": "json_object"}
+        )
+        result = result.choices[0].message.content
+
+        if check_blacklisted:
+            found_blacklisted_word = self._get_found_blacklisted_words(result)
+
+            if found_blacklisted_word is not None and try_count < self.TRY_LIMIT:
+                self._logger.warning("Result contains blacklisted words: " + str(found_blacklisted_word))
+                return await self._prediction(
+                    model, messages, token_count, fields_to_check, temperature, (try_count + 1), check_blacklisted
+                )
+            elif found_blacklisted_word is not None and try_count >= self.TRY_LIMIT:
+                return ""
+
+        if fields_to_check is None:
+            return json.loads(result)
+
+        if not self._check_fields(result, fields_to_check) and try_count < self.TRY_LIMIT:
+            return await self._prediction(
+                model, messages, token_count, fields_to_check, temperature, (try_count + 1), check_blacklisted
+            )
+
+        return json.loads(result)
+
+    async def prediction_override(self, **kwargs):
+        return await self._client.chat.completions.create(
+            **kwargs
+        )
+
+    @staticmethod
+    def _get_found_blacklisted_words(text: str):
+        text_lower = text.lower()
+        for word in BLACKLISTED_WORDS:
+            if re.search(r'\b' + re.escape(word) + r'\b', text_lower):
+                return word
+        return None
+
+    @staticmethod
+    def _count_total_tokens(messages):
+        total_tokens = 0
+        for message in messages:
+            total_tokens += count_tokens(message["content"])["n_tokens"]
+        return total_tokens
+
+    @staticmethod
+    def _check_fields(obj, fields):
+        return all(field in obj for field in fields)

app/services/impl/third_parties/whisper.py

@@ -0,0 +1,22 @@
+import os
+
+from fastapi.concurrency import run_in_threadpool
+
+from whisper import Whisper
+from app.services.abc import ISpeechToTextService
+
+
+class OpenAIWhisper(ISpeechToTextService):
+
+    def __init__(self, model: Whisper):
+        self._model = model
+
+    async def speech_to_text(self, file_path):
+        if os.path.exists(file_path):
+            result = await run_in_threadpool(
+                self._model.transcribe, file_path, fp16=False, language='English', verbose=False
+            )
+            return result["text"]
+        else:
+            print("File not found:", file_path)
+            raise Exception("File " + file_path + " not found.")