diff --git a/examples/__init__.py b/examples/__init__.py
index e69de29..966b406 100644
--- a/examples/__init__.py
+++ b/examples/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+Filename: __init__.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 16.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file is init file for examples package.
+"""
\ No newline at end of file
diff --git a/examples/image_generation/__init__.py b/examples/image_generation/__init__.py
index e69de29..5bfb53d 100644
--- a/examples/image_generation/__init__.py
+++ b/examples/image_generation/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+Filename: __init__.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 16.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file is init file for image_generation package.
+"""
diff --git a/examples/image_generation/dalle_test.py b/examples/image_generation/dalle_test.py
index 83e789b..53709c2 100644
--- a/examples/image_generation/dalle_test.py
+++ b/examples/image_generation/dalle_test.py
@@ -1,15 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+Filename: dalle_test.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 15.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file contains testing procedures for DALLE experiments
+"""
import asyncio
+from examples.creds import oai_token, oai_organization
from openai_api.src.openai_api import DALLE
-from creds import oai_token, oai_organization
-
-from time import sleep
dalle = DALLE(auth_token=oai_token, organization=oai_organization)
+
+
async def main():
- resp = await dalle.create_image_url('robocop (robot policeman, from 80s movie)')
+ """Main function for testing DALLE."""
+ resp = await dalle.create_image_url("robocop (robot policeman, from 80s movie)")
print(resp)
resp = await dalle.create_variation_from_url(resp[0])
print(resp)
-asyncio.run(main())
\ No newline at end of file
+
+asyncio.run(main())
diff --git a/examples/image_generation/gpt_functions.py b/examples/image_generation/gpt_functions.py
index 7d66131..17eacca 100644
--- a/examples/image_generation/gpt_functions.py
+++ b/examples/image_generation/gpt_functions.py
@@ -1,48 +1,70 @@
-import requests
-from PIL import Image
-from io import BytesIO
+# -*- coding: utf-8 -*-
+"""
+Filename: gpt_functions.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+Created: 15.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file contains testing functions for ChatGPT function calling using DALLE and Leonardo experiments
+"""
import json
+from io import BytesIO
-from creds import oai_token, oai_organization
+import requests
+from PIL import Image
+
+from examples.creds import oai_token, oai_organization
+from leonardo_api.src.leonardo_api.leonardo_sync import Leonardo
from openai_api.src.openai_api.dalle import DALLE
-from leonardo_api.leonardo_sync import Leonardo
-from page_retriever import PageRetriever
-from pytest_runner import run_tests
-doc_engine = PageRetriever('https://wwakabobik.github.io/')
+def get_weather(city, units):
+ """
+ Get the weather for a given city.
+ :param city: The city to get the weather for.
+ :param units: The units to use for the weather.
-def get_weather(city, units):
+ :return: The weather for the given city.
+ """
base_url = "http://api.openweathermap.org/data/2.5/weather"
- params = {
- "q": city,
- "appid": "93171b03384f92ee3c55873452a49c7c",
- "units": units
- }
+ params = {"q": city, "appid": "93171b03384f92ee3c55873452a49c7c", "units": units}
response = requests.get(base_url, params=params)
data = response.json()
return data
def get_current_weather(location, unit="metric"):
- """Get the current weather in a given location"""
+ """
+ Get the current weather in a given location
+
+ :param location: (str) The location to get the weather for.
+ :param unit: (str) The unit to use for the weather.
+ """
owm_info = get_weather(location, units=unit)
weather_info = {
"location": location,
"temperature": owm_info["main"]["temp"],
"unit": unit,
"forecast": owm_info["weather"][0]["description"],
- "wind": owm_info["wind"]["speed"]
+ "wind": owm_info["wind"]["speed"],
}
return json.dumps(weather_info)
def draw_image_using_dalle(prompt):
+ """
+ Draws image using user prompt. Returns url of image.
+
+ :param prompt: (str) Prompt, the description, what should be drawn and how
+ :return: (str) url of image
+ """
dalle = DALLE(auth_token=oai_token, organization=oai_organization)
image = dalle.create_image_url(prompt)
- url_dict = {'image_url': image[0]}
+ url_dict = {"image_url": image[0]}
response = requests.get(image[0])
img = Image.open(BytesIO(response.content))
img.show()
@@ -50,82 +72,60 @@ def draw_image_using_dalle(prompt):
def draw_image(prompt):
- leonardo = Leonardo(auth_token='a0178171-c67f-4922-afb3-458f24ecef1a')
+ """
+ Draws image using user prompt. Returns url of image.
+
+ :param prompt: (str) Prompt, the description, what should be drawn and how
+ :return: (dict) dict with url of image
+ """
+ leonardo = Leonardo(auth_token="a0178171-c67f-4922-afb3-458f24ecef1a")
leonardo.get_user_info()
- response = leonardo.post_generations(prompt=prompt, num_images=1, guidance_scale=5,
- model_id='e316348f-7773-490e-adcd-46757c738eb7', width=1024, height=768)
- response = leonardo.wait_for_image_generation(generation_id=response['sdGenerationJob']['generationId'])
- url_dict = {'image_url': response[0]['url']}
- response = requests.get(url_dict['image_url'])
+ response = leonardo.post_generations(
+ prompt=prompt,
+ num_images=1,
+ guidance_scale=5,
+ model_id="e316348f-7773-490e-adcd-46757c738eb7",
+ width=1024,
+ height=768,
+ )
+ response = leonardo.wait_for_image_generation(generation_id=response["sdGenerationJob"]["generationId"])
+ url_dict = {"image_url": response[0]["url"]}
+ response = requests.get(url_dict["image_url"])
img = Image.open(BytesIO(response.content))
img.show()
return json.dumps(url_dict)
gpt_functions = [
- {
- "name": "draw_image",
- "description": "Draws image using user prompt. Returns url of image.",
- "parameters": {
- "type": "object",
- "properties": {
- "prompt": {
- "type": "string",
- "description": "Prompt, the description, what should be drawn and how",
- },
+ {
+ "name": "draw_image",
+ "description": "Draws image using user prompt. Returns url of image.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "prompt": {
+ "type": "string",
+ "description": "Prompt, the description, what should be drawn and how",
},
- "required": ["prompt"],
},
+ "required": ["prompt"],
},
- {
- "name": "get_current_weather",
- "description": "Get the current weather in a given location",
- "parameters": {
- "type": "object",
- "properties": {
- "location": {
- "type": "string",
- "description": "The city and state, e.g. San Francisco, CA",
- },
- "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+ },
+ {
+ "name": "get_current_weather",
+ "description": "Get the current weather in a given location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state, e.g. San Francisco, CA",
},
- "required": ["location"],
+ "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
+ "required": ["location"],
},
- {
- "name": "get_page_code",
- "description": "Get page code to generate locators and tests",
- "parameters": {
- "type": "object",
- "properties": {
- "url": {
- "type": "string",
- "description": "The URL of the page to get the code from"
- }
- },
- "required": []
- }
- },
- {
- "name": "get_tests_results",
- "description": "Get the results of the tests",
- "parameters": {
- "type": "object",
- "properties": {
- "test_files": {
- "type": "array",
- "items": {
- "type": "string"
- },
- "description": "The list of test files to run"
- }
- },
- "required": []
- }
- }
- ]
-
-gpt_functions_dict = {'get_current_weather': get_current_weather,
- 'draw_image': draw_image,
- 'get_page_code': doc_engine.get_body_without_scripts,
- 'get_tests_results': run_tests('tests/test_example.py')}
\ No newline at end of file
+ },
+]
+
+gpt_functions_dict = {"get_current_weather": get_current_weather, "draw_image": draw_image}
diff --git a/examples/image_generation/test_leonardo.py b/examples/image_generation/test_leonardo.py
index ae841d4..6941aed 100644
--- a/examples/image_generation/test_leonardo.py
+++ b/examples/image_generation/test_leonardo.py
@@ -1,21 +1,38 @@
-import json
+# -*- coding: utf-8 -*-
+"""
+Filename: test_leonardo.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 15.10.2023
+Last Modified: 17.10.2023
+Description:
+This file contains testing procedures for Leonardo experiments
+"""
import asyncio
+import json
from leonardo_api import LeonardoAsync as Leonardo
async def main():
- leonardo = Leonardo(auth_token='a0178171-c67f-4922-afb3-458f24ecef1a')
+ """Main function"""
+ leonardo = Leonardo(auth_token="a0178171-c67f-4922-afb3-458f24ecef1a")
response = await leonardo.get_user_info()
print(response)
- response = await leonardo.post_generations(prompt="a beautiful necromancer witch resurrects skeletons against "
- "the backdrop of a burning ruined castle", num_images=1,
- negative_prompt='bright colors, good characters, positive',
- model_id='e316348f-7773-490e-adcd-46757c738eb7', width=1024, height=768,
- guidance_scale=3)
+ response = await leonardo.post_generations(
+ prompt="a beautiful necromancer witch resurrects skeletons against " "the backdrop of a burning ruined castle",
+ num_images=1,
+ negative_prompt="bright colors, good characters, positive",
+ model_id="e316348f-7773-490e-adcd-46757c738eb7",
+ width=1024,
+ height=768,
+ guidance_scale=3,
+ )
print(response)
- response = await leonardo.wait_for_image_generation(generation_id=response['sdGenerationJob']['generationId'])
- print(json.dumps(response[0]['url']))
+ response = await leonardo.wait_for_image_generation(generation_id=response["sdGenerationJob"]["generationId"])
+ print(json.dumps(response[0]["url"]))
+
asyncio.run(main())
diff --git a/examples/speak_and_hear/__init__.py b/examples/speak_and_hear/__init__.py
index e69de29..54f45a9 100644
--- a/examples/speak_and_hear/__init__.py
+++ b/examples/speak_and_hear/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+Filename: __init__.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 16.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file is init file for speak_and_hear package.
+"""
diff --git a/examples/speak_and_hear/test_gpt.py b/examples/speak_and_hear/test_gpt.py
index a5cd48d..b273626 100644
--- a/examples/speak_and_hear/test_gpt.py
+++ b/examples/speak_and_hear/test_gpt.py
@@ -5,30 +5,26 @@
Copyright (c) 2023. All rights reserved.
Created: 25.08.2023
-Last Modified: 25.08.2023
+Last Modified: 17.10.2023
Description:
-This file contains testing procedures for ChatGPt experiments
+This file contains testing procedures for ChatGPT experiments
"""
+import asyncio
import string
import sys
-import asyncio
-
from utils.audio_recorder import AudioRecorder
from utils.transcriptors import CustomTranscriptor
from utils.tts import CustomTTS
-
from ..creds import oai_token, oai_organization
from ...openai_api import ChatGPT
-
gpt = ChatGPT(auth_token=oai_token, organization=oai_organization, model="gpt-3.5-turbo")
gpt.max_tokens = 200
gpt.stream = True
-
tts = CustomTTS(method="google", lang="en")
# queues
@@ -37,6 +33,13 @@
async def ask_chat(user_input):
+ """
+ Ask chatbot a question
+
+ :param user_input: (str) User input
+
+ :return: (str) Chatbot response
+ """
full_response = ""
word = ""
async for response in gpt.str_chat(user_input):
@@ -54,6 +57,7 @@ async def ask_chat(user_input):
async def tts_task():
+ """Task to process words and chars for TTS"""
limit = 5
empty_counter = 0
while True:
@@ -84,6 +88,7 @@ async def tts_task():
async def tts_sentence_task():
+ """Task to handle sentences for TTS"""
punctuation_marks = ".?!,;:"
sentence = ""
while True:
@@ -99,6 +104,7 @@ async def tts_sentence_task():
async def tts_worker():
+ """Task to process sentences for TTS"""
while True:
try:
sentence = await tts_queue.get()
@@ -110,6 +116,7 @@ async def tts_worker():
async def get_user_input():
+ """Get user input"""
while True:
try:
user_input = input()
@@ -122,6 +129,7 @@ async def get_user_input():
async def main():
+ """Main function"""
asyncio.create_task(tts_sentence_task())
asyncio.create_task(tts_worker())
method = "google"
@@ -137,12 +145,11 @@ async def main():
pass
if transcript:
print(f"User: {transcript}")
- #translate = CustomTranslator(source='ru', target='en').translate(transcript)
- #print(translate)
+ # translate = CustomTranslator(source='ru', target='en').translate(transcript)
+ # print(translate)
response = await ask_chat(transcript)
except KeyboardInterrupt:
break
asyncio.run(main())
-
diff --git a/examples/test_generator/__init__.py b/examples/test_generator/__init__.py
index e69de29..59327fe 100644
--- a/examples/test_generator/__init__.py
+++ b/examples/test_generator/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+Filename: __init__.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 16.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file is init file for test_generator package.
+"""
diff --git a/examples/test_generator/generator_test.py b/examples/test_generator/generator_test.py
index d308896..8693fa3 100644
--- a/examples/test_generator/generator_test.py
+++ b/examples/test_generator/generator_test.py
@@ -1,29 +1,28 @@
# -*- coding: utf-8 -*-
"""
-Filename: __generator_test__.py
+Filename: generator_test.py
Author: Iliya Vereshchagin
Copyright (c) 2023. All rights reserved.
Created: 16.10.2023
-Last Modified: 16.10.2023
+Last Modified: 17.10.2023
Description:
-This file contains testing procedures for ChatGPt experiments
+This file contains testing procedures for ChatGPT experiments
"""
+import asyncio
import json
import logging
-import asyncio
-
-from openai_api.src.openai_api import ChatGPT
-from openai_api.src.openai_api.logger_config import setup_logger
from examples.creds import oai_token, oai_organization
-from examples.test_generator.pom_case_generator import PomTestCaseGenerator
from examples.test_generator.gpt_functions import gpt_functions, gpt_functions_dict
+from examples.test_generator.pom_case_generator import PomTestCaseGenerator
+from openai_api.src.openai_api import ChatGPT
+from openai_api.src.openai_api.logger_config import setup_logger
-generator = PomTestCaseGenerator(url='https://www.saucedemo.com/')
-#generator = PomTestCaseGenerator(url='https://automationintesting.com/selenium/testpage/')
+generator = PomTestCaseGenerator(url="https://www.saucedemo.com/")
+# generator = PomTestCaseGenerator(url='https://automationintesting.com/selenium/testpage/')
system_instructions = """
@@ -66,30 +65,32 @@ def setup_gpt():
gpt.logger = setup_logger("gpt", "gpt.log", logging.INFO)
gpt.system_settings = ""
gpt.function_dict = gpt_functions_dict
- gpt.function_call = 'auto'
+ gpt.function_call = "auto"
gpt.functions = gpt_functions
gpt.system_settings = system_instructions
return gpt
async def main():
+ """Main function for testing GPT bot"""
print("===Setup GPT bot===")
gpt = setup_gpt()
print("===Get page code of https://www.saucedemo.com/ and generate POM and tests===")
response = await anext(gpt.str_chat("Get page code of https://www.saucedemo.com/ and generate POM and tests"))
print(response)
- response = response.replace('\n', '')
- generator.create_files_from_json(json.loads(response),
- pom_folder='examples/test_generator/pom',
- tests_folder='examples/test_generator/tests')
+ response = response.replace("\n", "")
+ generator.create_files_from_json(
+ json.loads(response), pom_folder="examples/test_generator/pom", tests_folder="examples/test_generator/tests"
+ )
print("===Get tests results for examples/test_generator/tests/test_index.py==")
response = await anext(gpt.str_chat("Get tests results for examples/test_generator/tests/test_index.py"))
print(response)
print("===If there are failures in code, please fix it by fixing POM and tests===")
response = await anext(gpt.str_chat("If there are failures in code, please fix it by fixing POM and tests"))
print(response)
- generator.create_files_from_json(json.loads(response),
- pom_folder='..pom',
- tests_folder='examples/test_generator/tests')
+ generator.create_files_from_json(
+ json.loads(response), pom_folder="..pom", tests_folder="examples/test_generator/tests"
+ )
+
asyncio.run(main())
diff --git a/examples/test_generator/gpt_functions.py b/examples/test_generator/gpt_functions.py
index 494a3bf..2f00501 100644
--- a/examples/test_generator/gpt_functions.py
+++ b/examples/test_generator/gpt_functions.py
@@ -14,7 +14,7 @@
from examples.test_generator.pytest_runner import run_tests
from utils.page_retriever import PageRetriever
-doc_engine = PageRetriever("https://wwakabobik.github.io/")
+doc_engine = PageRetriever()
gpt_functions = [
{
"name": "get_page_code",
diff --git a/examples/test_generator/pom/__init__.py b/examples/test_generator/pom/__init__.py
index e69de29..e4e90c0 100644
--- a/examples/test_generator/pom/__init__.py
+++ b/examples/test_generator/pom/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+Filename: __init__.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 16.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file is init file for test pom package.
+"""
diff --git a/examples/test_generator/pom_case_generator.py b/examples/test_generator/pom_case_generator.py
index 0561407..ce01b8f 100644
--- a/examples/test_generator/pom_case_generator.py
+++ b/examples/test_generator/pom_case_generator.py
@@ -10,7 +10,7 @@ def __init__(self, url=''):
"""
General init.
- :param url: URL of the page.
+ :param url: (str) URL of the page.
"""
self.url = url
@@ -18,7 +18,7 @@ def set_url(self, url):
"""
Set the url.
- :param url: URL of the page.
+ :param url: (str) URL of the page.
"""
self.url = url
@@ -26,10 +26,10 @@ def ___create_pom_file(self, file_name, page_objects, url='', pom_folder='pom'):
"""
Create page object model file.
- :param file_name: Name of the file.
- :param page_objects: List of page objects.
- :param url: URL of the page.
- :param pom_folder: Folder for page object model files.
+ :param file_name: (str) Name of the file.
+ :param page_objects: (list) List of page objects.
+ :param url: (str) URL of the page.
+ :param pom_folder: (str) Folder for page object model files.
"""
if not url:
url = self.url
@@ -51,10 +51,10 @@ def ___create_test_file(file_name, tests, pom_folder='pom', tests_folder='tests'
"""
Create test file.
- :param file_name: Name of the file.
- :param tests: List of tests.
- :param pom_folder: Folder for page object model files.
- :param tests_folder: Folder for test files.
+ :param file_name: (str) Name of the file.
+ :param tests: (list) List of tests.
+ :param pom_folder: (str) Folder for page object model files.
+ :param tests_folder: (str) Folder for test files.
"""
with open(f'{tests_folder}/test_{file_name}.py', 'w') as test_file:
test_file.write('import pytest\n\n')
@@ -72,10 +72,10 @@ def create_files_from_json(self, json_data, url='', pom_folder='pom', tests_fold
"""
Create test and page object model files from json data.
- :param json_data: JSON data.
- :param url: URL of the page.
- :param pom_folder: Folder for page object model files.
- :param tests_folder: Folder for test files.
+ :param json_data: (str) JSON data.
+ :param url: (str) URL of the page.
+ :param pom_folder: (str) Folder for page object model files.
+ :param tests_folder: (str) Folder for test files.
"""
if not url:
url = self.url
diff --git a/examples/test_generator/pytest_runner.py b/examples/test_generator/pytest_runner.py
index 285e933..6b922b5 100644
--- a/examples/test_generator/pytest_runner.py
+++ b/examples/test_generator/pytest_runner.py
@@ -12,15 +12,12 @@ def run_tests(test_files, add_failed_html=True, add_failure_reasons=True, count_
"""
Run tests and return results in JSON format.
- Args:
- test_files: list with test files.
- add_failed_html: boolean to add html report.
- add_failure_reasons: boolean to add failure reasons.
- count_of_htmls: count of htmls to add. Doesn't recommend to use more than 1.
-
- Returns:
- JSON with results.
+ :param test_files: (list) list with test files.
+ :param add_failed_html: (bool) boolean to add html report.
+ :param add_failure_reasons: (bool) boolean to add failure reasons.
+ :param count_of_htmls: (int) count of htmls to add. Doesn't recommend to use more than 1.
+ :return: JSON with results.
"""
pytest.main(["-q", "--json-report", "--json-report-file=test_report.json", "-n=4", "-rfEx --tb=none -p no:warnings -p no:logging"] + test_files)
@@ -56,11 +53,9 @@ def parse_error_page(node_name):
"""
Parse error page.
- Args:
- node_name: name of the node.
+ :param node_name: (str) name of the node.
- Returns:
- string with parsed page.
+ :return: (str) formatted content of the page.
"""
parser = PageRetriever()
try:
diff --git a/examples/test_generator/tests/conftest.py b/examples/test_generator/tests/conftest.py
index a7e29b7..0467cdd 100644
--- a/examples/test_generator/tests/conftest.py
+++ b/examples/test_generator/tests/conftest.py
@@ -1,5 +1,15 @@
-from time import sleep
-
+# -*- coding: utf-8 -*-
+"""
+Filename: conftest.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 15.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file contains pytest fixtures for tests
+"""
import pytest
from selenium import webdriver
@@ -9,6 +19,12 @@
def pytest_runtest_makereport(item, call):
+ """
+ Pytest hook for saving html page on test failure
+
+ :param item: pytest item
+ :param call: pytest call
+ """
if "driver" in item.fixturenames:
web_driver = item.funcargs["driver"]
if call.when == "call" and call.excinfo is not None:
@@ -17,7 +33,12 @@ def pytest_runtest_makereport(item, call):
@pytest.fixture
-def driver(request):
+def driver():
+ """
+ Pytest fixture for selenium webdriver
+
+ :return: webdriver
+ """
options = Options()
options.add_argument("--headless")
options.headless = True
diff --git a/utils/__init__.py b/utils/__init__.py
index e69de29..1672f66 100644
--- a/utils/__init__.py
+++ b/utils/__init__.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+"""
+Filename: __init__.py.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 16.10.2023
+Last Modified: 17.10.2023
+
+Description:
+This file is init file for utils package.
+"""
+from .page_retriever import PageRetriever
+from .tts import CustomTTS
+from .transcriptors import CustomTranscriptor
+from .translators import CustomTranslator
+from .audio_recorder import AudioRecorder, record_and_convert_audio
+from .logger_config import setup_logger
+from .other import is_heroku_environment
diff --git a/utils/article_extractor.py b/utils/article_extractor.py
index 98dc334..ceed070 100644
--- a/utils/article_extractor.py
+++ b/utils/article_extractor.py
@@ -5,7 +5,7 @@
Copyright (c) 2023. All rights reserved.
Created: 25.08.2023
-Last Modified: 12.09.2023
+Last Modified: 17.10.2023
Description:
This file contains implementation for Article Extractor from internet page
@@ -14,18 +14,17 @@
import requests
from readability import Document
+
# FIXME: This is a temporary solution. We need to find a better way to extract
def get_content(url):
"""
This function extracts content from internet page.
- Args:
- url: URL of internet page.
- Returns:
- Content of internet page.
+ :param url: The URL of the page to extract content from.
+ :return: The content of the page.
"""
session = requests.Session()
response = session.get(url)
diff --git a/utils/audio_recorder.py b/utils/audio_recorder.py
index ee50285..dbea0d6 100644
--- a/utils/audio_recorder.py
+++ b/utils/audio_recorder.py
@@ -5,13 +5,12 @@
Copyright (c) 2023. All rights reserved.
Created: 25.08.2023
-Last Modified: 26.08.2023
+Last Modified: 17.10.2023
Description:
This file contains implementation for Audio Recorder
"""
-import math
import os
import struct
import tempfile
@@ -19,6 +18,7 @@
import uuid
import wave
+import math
import pyaudio
import sounddevice as sd
import soundfile as sf
@@ -33,8 +33,8 @@ def record_and_convert_audio(duration: int = 5, frequency_sample: int = 16000):
The audio is then saved as a temporary .wav file, converted to .mp3 format, and the .wav file is deleted.
The function returns the path to the .mp3 file.
- :param duration: The duration of the audio recording in seconds. Default is 5 seconds.
- :param frequency_sample: The frequency sample rate of the audio recording. Default is 16000 Hz.
+ :param duration: (int) The duration of the audio recording in seconds. Default is 5 seconds.
+ :param frequency_sample: (int) The frequency sample rate of the audio recording. Default is 16000 Hz.
:return: The path to the saved .mp3 file.
"""
diff --git a/utils/logger_config.py b/utils/logger_config.py
index 8f82425..f0a3e5c 100644
--- a/utils/logger_config.py
+++ b/utils/logger_config.py
@@ -5,7 +5,7 @@
Copyright (c) 2023. All rights reserved.
Created: 25.08.2023
-Last Modified: 25.08.2023
+Last Modified: 17.10.2023
Description:
This file contains configuration for loggers.
@@ -26,7 +26,6 @@ def setup_logger(name: str, log_file: str, level=logging.DEBUG):
:param level: logging level. Default is logging.DEBUG
:returns: logger object
-
"""
formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")
diff --git a/utils/other.py b/utils/other.py
index 7111eba..b71e93c 100644
--- a/utils/other.py
+++ b/utils/other.py
@@ -5,7 +5,7 @@
Copyright (c) 2023. All rights reserved.
Created: 25.08.2023
-Last Modified: 25.08.2023
+Last Modified: 17.10.2023
Description:
This file contains several other stuff.
@@ -18,10 +18,6 @@ def is_heroku_environment():
"""
Check current env - are we on Heroku or not
- Args:
- - None
-
- Returns:
- - bool: True is current environment is Heroku, otherwise - False.
+ :return: True if we are on Heroku, False otherwise
"""
return "DYNO" in os.environ and "PORT" in os.environ
diff --git a/utils/page_retriever.py b/utils/page_retriever.py
index f80baee..850dcf3 100644
--- a/utils/page_retriever.py
+++ b/utils/page_retriever.py
@@ -1,24 +1,36 @@
-"""PageRetriever class for extracting the page content from the url."""
+# -*- coding: utf-8 -*-
+"""
+Filename: page_retriever.py
+Author: Iliya Vereshchagin
+Copyright (c) 2023. All rights reserved.
+
+Created: 30.09.2023
+Last Modified: 17.10.2023
+
+Description:
+This module contains implementation for PageRetriever
+"""
import re
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service as ChromeService
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
class PageRetriever:
"""The PageRetriever class is for managing an instance of the PageRetriever."""
- def __init__(self, url=''):
+
+ def __init__(self, url=""):
"""
General init.
- :param url: URL of the page.
+ :param url: (str) URL of the page.
"""
options = Options()
options.add_argument("--headless")
@@ -31,7 +43,7 @@ def set_url(self, url):
"""
Set the url.
- :param url: URL of the page.
+ :param url: (str) URL of the page.
"""
self.url = url
@@ -39,8 +51,8 @@ def get_page(self, url=None):
"""
Get the page content from the url.
- :param url: URL of the page.
- :returns: HTML content of the page.
+ :param url: (str) URL of the page.
+ :return: (str) HTML content of the page.
"""
if url:
self.set_url(url)
@@ -50,8 +62,8 @@ def get_body(self, url=None):
"""
Get the body content of the page.
- :param url: URL of the page.
- :returns: Body content of the page.
+ :param url: (str) URL of the page.
+ :return: (str) Body content of the page.
"""
if url:
self.set_url(url)
@@ -61,8 +73,9 @@ def get_body_without_scripts(self, url=None):
"""
Get the body content of the page without tags.
- :param url: URL of the page.
- :returns: Body content of the page without tags.
+ :param url: (str) URL of the page.
+
+ :return: (str) Body content of the page without tags.
"""
if url:
self.set_url(url)
@@ -72,12 +85,13 @@ def get_page_content(self, url):
"""
Get the page content from the url.
- :param url: URL of the page.
- :returns: HTML content of the page.
+ :param url: (str) URL of the page.
+
+ :return: (str) HTML content of the page.
"""
self.driver.get(url)
- WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, 'body')))
+ WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
start_time = time.time()
while True:
@@ -99,10 +113,11 @@ def extract_body_content(html_content):
"""
Extract the body content from the html_content.
- :param html_content: HTML content of the page.
- :returns: Body content of the page.
+ :param html_content: (str) HTML content of the page.
+
+ :return: (str) Body content of the page.
"""
- soup = BeautifulSoup(html_content, 'html.parser')
+ soup = BeautifulSoup(html_content, "html.parser")
body_content = soup.body
return str(body_content)
@@ -112,11 +127,12 @@ def remove_script_tags(input_content):
"""
Remove all tags from the input_content.
- :param input_content: HTML content of the page.
- :returns: Body content of the page without tags.
+ :param input_content: (str) HTML content of the page.
+
+ :return: (str) Body content of the page without tags.
"""
- pattern_1 = re.compile(r'.*?', re.DOTALL)
- pattern_2 = re.compile(r'.*?', re.DOTALL)
- output = re.sub(pattern_1, '', input_content)
- output = re.sub(pattern_2, '', output)
+ pattern_1 = re.compile(r".*?", re.DOTALL)
+ pattern_2 = re.compile(r".*?", re.DOTALL)
+ output = re.sub(pattern_1, "", input_content)
+ output = re.sub(pattern_2, "", output)
return output
diff --git a/utils/transcriptors.py b/utils/transcriptors.py
index 9661a88..a4042c6 100644
--- a/utils/transcriptors.py
+++ b/utils/transcriptors.py
@@ -5,7 +5,7 @@
Copyright (c) 2023. All rights reserved.
Created: 25.08.2023
-Last Modified: 25.08.2023
+Last Modified: 17.10.2023
Description:
This module contains implementation for Custom Transcriptor
@@ -15,9 +15,7 @@
class CustomTranscriptor:
- """
- This is wrapper class for Google Transcriptor which uses microphone to get audio sample.
- """
+ """This is wrapper class for Google Transcriptor which uses microphone to get audio sample."""
def __init__(self, language="en-EN"):
"""
diff --git a/utils/translators.py b/utils/translators.py
index c4d6ec9..78c63ba 100644
--- a/utils/translators.py
+++ b/utils/translators.py
@@ -5,7 +5,7 @@
Copyright (c) 2023. All rights reserved.
Created: 25.08.2023
-Last Modified: 25.08.2023
+Last Modified: 17.10.2023
Description:
This module contains implementation for Custom Translator
@@ -15,9 +15,7 @@
class CustomTranslator(GoogleTranslator):
- """
- This class implements wrapper for GoogleTranslator
- """
+ """This class implements wrapper for GoogleTranslator"""
def __init__(self, source, target, **kwargs):
"""
@@ -69,6 +67,7 @@ def target(self, value):
def translate(self, text: str, **kwargs) -> str:
"""
+ This function translates text from source language to target language.
:param text: Text (string) to translate.
:param kwargs: Custom arguments, optional.