Skip to content

Commit

Permalink
get_api_key_while_testing
Browse files Browse the repository at this point in the history
  • Loading branch information
SermetPekin committed Apr 17, 2024
1 parent 83bebf0 commit 868e49d
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 33 deletions.
102 changes: 87 additions & 15 deletions evdspy/EVDSlocal/index_requests/user_requests.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@

# ...........................................................................................
from evdspy import setup
from evdspy.EVDSlocal.components.excel_class import replace_all
from evdspy.EVDSlocal.index_requests.get_series_indexes_utils import (
default_start_date_fnc,
default_end_date_fnc, correct_types,
)
from evdspy.EVDSlocal.utils.utils_test import get_api_key_while_testing
from evdspy.EVDSlocal.utils.utils_test import get_api_key_while_testing, ApiClassWhileTesting
from evdspy.EVDSlocal.index_requests.index_util_funcs import json_to_df, make_df_float
from evdspy.EVDSlocal.requests_.real_requests import *
from evdspy.EVDSlocal.utils.github_actions import PytestTesting, GithubActions
Expand All @@ -23,11 +22,14 @@
import pandas as pd
import json
from abc import ABC


# ....................................................................... ProxyManager
@dataclass
class ProxyManager:
proxy: Optional[str] = None
proxies: Optional[dict[Any, Any]] = None

def get_proxies(self) -> Optional[dict[Any, Any]]:
if self.proxies is None:
if self.proxy is None:
Expand All @@ -37,34 +39,44 @@ def get_proxies(self) -> Optional[dict[Any, Any]]:
else:
proxies = self.proxies
return proxies

def get_proxies_helper(self) -> Optional[dict[Any, Any]]:
if self.proxy is None:
return None
proxy = self.proxy
proxies = {
'http': proxy,
'https': proxy,
'http': proxy,
'https': proxy,
}
return proxies


# ....................................................................... Serialize
class Serialize(ABC):
"""To check whether two Config Requests are perfect substitutes """

def serialize(self):
return json.dumps(
self,
default=lambda o: o.__dict__,
sort_keys=True,
indent=4)
self,
default=lambda o: o.__dict__,
sort_keys=True,
indent=4)

@property
def hash(self) -> str:
import hashlib
return str(int(hashlib.sha256(self.serialize().encode('utf-8')).hexdigest(), 16) % 10 ** 8)

def __eq__(self, other):
return self.hash == other.hash


def looks_like_datagroup(string: Any):
if not isinstance(string, str):
return False
return 'bie_' in string


# ....................................................................... RequestConfig
def clean_chars(string: str):
import re
Expand All @@ -74,11 +86,15 @@ def clean_chars(string: str):
string = string.replace("_", ".")
string = re.sub('[^0-9a-zA-Z.]+', '', string)
return string


def test_clean_chars(capsys):
with capsys.disabled():
assert clean_chars("AAA_BBB?*-()$? ") == "AAA.BBB"
assert clean_chars("AAA..._BBB?*-()$? ") == "AAA....BBB"
assert clean_chars("bie_BBB?*-()$? ") == "bie_BBB"


@dataclass
class RequestConfig(Serialize):
index: Union[str, tuple[str, ...], list[str, ...]]
Expand Down Expand Up @@ -111,24 +127,29 @@ class RequestConfig(Serialize):
Raises:
ValueError: If the index is in an invalid format or necessary attributes are missing.
"""

def check_type(self):
if looks_like_datagroup(self.initial_index):
return "datagroup"
return "series"

def __post_init__(self):
self.initial_index = self.index
self.correct_index()
self.correct_formulas_aggr()
self.check()

def correct_index(self):
if "\n" in self.index or "\t" in self.index:
self.index = self.template_to_tuple(self.index)
self.index = tuple([self.index]) if not isinstance(self.index, (tuple, list,)) else self.index
self.index = tuple(map(clean_chars, self.index))
self.check_index()

def correct_formulas_aggr(self):
self.formulas = self.correct_type_to_tuple(self.formulas)
self.aggregation = self.correct_type_to_tuple(self.aggregation)

def correct_type_to_tuple(self, value: any) -> Optional[tuple]:
if value is None:
return None
Expand All @@ -138,62 +159,79 @@ def correct_type_to_tuple(self, value: any) -> Optional[tuple]:
if isinstance(value, (str,)):
return tuple(value for _ in self.index)
return tuple(value[0] for _ in self.index)

@staticmethod
def template_to_tuple(index: str) -> tuple:
def clean(string: str):
return string.split("#")[0].strip() if len(string.split("#")) > 0 else None

index = replace_all(index, "\t", "\n")
index_tuple = index.splitlines()
t = tuple(clean(x) for x in index_tuple)
return tuple(x for x in t if x is not None and len(x) > 3)

def check_index(self) -> None:
if isinstance(self.index, (int, float,)):
raise ValueError("index must be a string ot tuple of string ")

def freq_str(self) -> str:
if self.frequency is None:
return ""
if isinstance(self.frequency, int):
return f"&frequency={self.frequency}"
return freq_enum(self.frequency)

def agr_form_type_to_str(self, value: Optional[tuple], part_name="aggregationTypes"):
if value is None:
return ""
value = tuple(map(str, value))
string = "-".join(value)
return f"&{part_name}=" + string

def aggregation_type_to_str(self) -> str:
self.aggregation = correct_types(self.aggregation, enum_class=AggregationType)
return self.agr_form_type_to_str(self.aggregation, "aggregationTypes")

def formulas_to_str(self) -> str:
self.formulas = correct_types(self.formulas, enum_class=Formulas)
return self.agr_form_type_to_str(self.formulas, "formulas")

def check(self) -> None:
if self.formulas is not None:
assert len(self.formulas) == len(self.index)
if self.aggregation is not None:
assert len(self.aggregation) == len(self.index)
if self.frequency is not None:
assert isinstance(self.frequency, (int, str,))

def create_series_part(self) -> str:
indexes = self.index
if isinstance(indexes, str):
indexes = tuple([indexes])
return "-".join(indexes)


# ....................................................................... UrlSeries
class UrlSeries:
@property
def domain(self) -> str:
return "https://evds2.tcmb.gov.tr/service/evds"

@property
def alias(self):
return "series="


class UrlDataGroup(UrlSeries):
@property
def domain(self) -> str:
return "https://evds2.tcmb.gov.tr/service/evds"

@property
def alias(self):
return "datagroup="


# ....................................................................... UrlBuilder
class UrlBuilder:
def __init__(self,
Expand All @@ -204,21 +242,26 @@ def __init__(self,
if not url_type:
self.get_url_type()
self.alias = self.url_type.alias

def get_url_type(self):
url_type = UrlSeries()
if self.config.check_type() == "datagroup":
url_type = UrlDataGroup()
self.url_type = url_type

def create_url_for_series(cls) -> str:
domain = cls.domain
return f"{domain}/{cls.alias}{cls.series_part}&startDate={cls.config.start_date}&endDate={cls.config.end_date}&type=json"

@property
def domain(self) -> str:
return self.url_type.domain

@property
def basic_url(self) -> str:
config = self.config
return f"{self.domain}/{self.alias}{self.series_part}&startDate={config.start_date}&endDate={config.end_date}&type=json"

@property
def url(self) -> str:
config = self.config
Expand All @@ -230,21 +273,28 @@ def url(self) -> str:
freq_string = config.freq_str()
"""..."""
parts = (
f"{self.domain}/{self.alias}{self.series_part}{freq_string}{formulas_str}{aggregation_type_str}",
f"startDate={config.start_date}",
f"endDate={config.end_date}",
"type=json"
f"{self.domain}/{self.alias}{self.series_part}{freq_string}{formulas_str}{aggregation_type_str}",
f"startDate={config.start_date}",
f"endDate={config.end_date}",
"type=json"
)
return "&".join(parts)


def create_cache_version(fnc: Callable):
@MyCache().cache
def fnc_cache(*args, **kw):
return fnc(*args, **kw)

return fnc_cache


def cache_or_raw_fnc(fnc, cache=False):
if not cache:
return fnc
return create_cache_version(fnc)


def test_cache_or_raw_fnc(capsys):
with capsys.disabled():
setup()
Expand All @@ -254,17 +304,22 @@ def test_cache_or_raw_fnc(capsys):
assert f(3) == 9
assert f2(3) == 9
assert id(f2) != id(f)


class ApiRequester:
def __init__(self, url_builder: UrlBuilder, proxy_manager: ProxyManager):
self.url_builder = url_builder
self.proxy_manager = proxy_manager
self.proxies = self.proxy_manager.get_proxies()
self.url = self.url_builder.url
self.response = None

def get(self):
return self.request()

def __call__(self, *args, **kwargs):
return self.get()

def dry_request(self) -> RequestConfig:
api_key = self.get_api_key(check=False)
print(f"""
Expand All @@ -278,9 +333,11 @@ def dry_request(self) -> RequestConfig:
---------------------------------
""")
return self.url_builder.config

def is_response_ok(self, response):
return isinstance(response, requests.Response) \
and response.status_code == 200

@staticmethod
def obscure(string: str):
if not isinstance(string, str):
Expand All @@ -292,12 +349,18 @@ def obscure(string: str):
else:
strings.append(i)
return "".join(strings)

def get_api_key(self, check=True) -> str:
if PytestTesting().is_testing():
api_key = get_api_key_while_testing()
if PytestTesting().is_testing() or GithubActions().is_testing():
api_key = self.get_api_key_while_testing()
else:
api_key = ApikeyClass().get_valid_api_key(check=check)
return api_key

def get_api_key_while_testing(self):

return ApiClassWhileTesting()()

def request(self) -> Any:
api_key = self.get_api_key()
if api_key is False:
Expand All @@ -306,11 +369,13 @@ def request(self) -> Any:
if PytestTesting().is_testing():
raise NotImplementedError
proxies = self.proxy_manager.get_proxies()

def local_request(url: str) -> requests.Response:
requester = RealRequestWithParam(url,
proxies=proxies,
api_key=api_key)
return requester.request()

request_func = cache_or_raw_fnc(local_request,
cache=self.url_builder.config.cache)
response = False
Expand All @@ -322,6 +387,7 @@ def local_request(url: str) -> requests.Response:
if not self.is_response_ok(response):
raise HTTPError(response=response)
return response.json()

def is_ok(self) -> bool:
response = self.response
ok = isinstance(response, requests.Response) and response.status_code == 200
Expand All @@ -331,10 +397,15 @@ def is_ok(self) -> bool:
else:
print("<data is here>")
return ok


import traceback


class DataProcessor:
def __init__(self, data: Any):
self.data = data

def process_to_dataframe(self) -> Optional[pd.DataFrame]:
try:
df = json_to_df(self.data)
Expand All @@ -345,5 +416,6 @@ def process_to_dataframe(self) -> Optional[pd.DataFrame]:
if isinstance(df, pd.DataFrame):
df = make_df_float(df)
return df

def __call__(self, *args, **kwargs) -> pd.DataFrame:
return self.process_to_dataframe()
return self.process_to_dataframe()
16 changes: 0 additions & 16 deletions evdspy/EVDSlocal/tests/test_user_requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,23 +158,7 @@ def test_get_series2(capsys):
import os


class ApiClassWhileTesting():
"""ApiClassWhileTesting"""

def __init__(self):
self.api_key = self.get_api_key()

def get_api_key(self):
if GithubActions().is_testing():
return os.getenv("EVDS_API_KEY")
return get_api_key_while_testing()

@property
def key(self):
return self.api_key

def __call__(self, *args, **kwargs):
return self.key


def test_get_api_key_while_testing(capsys):
Expand Down
Loading

0 comments on commit 868e49d

Please sign in to comment.