Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Scheduled queries google dbm #158

Draft
wants to merge 7 commits into
base: dev
Choose a base branch
from
Draft

Scheduled queries google dbm #158

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
42d3e77
Scheduled queries google dbm
Jul 5, 2021
324aadc
add tests scheduled query
Jul 5, 2021
88676a2
reformatting
Jul 26, 2021
22b5b42
update test reader dbm
Jul 26, 2021
4a0f442
fix test
Jul 26, 2021
5c75750
update doc reader
Jul 26, 2021
4f7f5a6
possible values in json config
Jul 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions ack/readers/google_dbm/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

import click
from ack.readers.google_dbm.config import POSSIBLE_REQUEST_TYPES
from ack.readers.google_dbm.config import POSSIBLE_REQUEST_TYPES, POSSIBLE_FREQUENCIES, POSSIBLE_TIMEZONE_CODES
from ack.readers.google_dbm.reader import GoogleDBMReader
from ack.utils.args import extract_args
from ack.utils.processor import processor
Expand All @@ -33,7 +33,8 @@
@click.option("--dbm-request-type", type=click.Choice(POSSIBLE_REQUEST_TYPES), required=True)
@click.option("--dbm-query-id")
@click.option("--dbm-query-title")
@click.option("--dbm-query-frequency", default="ONE_TIME")
@click.option("--dbm-query-frequency", type=click.Choice(POSSIBLE_FREQUENCIES), default="ONE_TIME")
@click.option("--dbm-query-timezone-code", type=click.Choice(POSSIBLE_TIMEZONE_CODES), default="America/New_York")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add this new option to the documentation ? A small guide on how to use sphinx is available here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes thank you

@click.option("--dbm-query-param-type", default="TYPE_TRUEVIEW")
@click.option("--dbm-start-date", type=click.DateTime())
@click.option("--dbm-end-date", type=click.DateTime())
Expand Down
17 changes: 17 additions & 0 deletions ack/readers/google_dbm/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,26 @@
GOOGLE_TOKEN_URI = "https://accounts.google.com/o/oauth2/token"

DAY_RANGES = ("PREVIOUS_DAY", "LAST_30_DAYS", "LAST_90_DAYS", "LAST_7_DAYS", "PREVIOUS_MONTH", "PREVIOUS_WEEK")
POSSIBLE_FREQUENCIES = ("DAILY", "MONTHLY", "ONE_TIME", "QUARTERLY", "SEMI_MONTHLY", "WEEKLY")
POSSIBLE_TIMEZONE_CODES = (
"Africa/Johannesburg",
"America/Los_Angeles",
"America/New_York",
"America/Sao_Paulo",
"Asia/Dubai",
"Asia/Hong_Kong",
"Asia/Jerusalem",
"Asia/Shanghai",
"Asia/Tokyo",
"Australia/Sydney",
"Europe/London",
"Europe/Paris",
"Pacific/Auckland",
)
POSSIBLE_REQUEST_TYPES = [
"existing_query",
"custom_query",
"custom_scheduled_query",
"existing_query_report",
"custom_query_report",
"lineitems_objects",
Expand Down
35 changes: 26 additions & 9 deletions ack/readers/google_dbm/reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,12 @@ def __init__(self, access_token, refresh_token, client_secret, client_id, **kwar

self.kwargs = kwargs

check_date_range_definition_conformity(
self.kwargs.get("start_date"), self.kwargs.get("end_date"), self.kwargs.get("day_range")
)
if self.kwargs.get("query_frequency") == "ONE_TIME":
check_date_range_definition_conformity(
self.kwargs.get("start_date"), self.kwargs.get("end_date"), self.kwargs.get("day_range")
)
else:
check_date_range_definition_conformity(self.kwargs.get("start_date"), self.kwargs.get("end_date"), None)
benoitbazouin marked this conversation as resolved.
Show resolved Hide resolved

def get_query(self, query_id):
if query_id:
Expand All @@ -73,7 +76,8 @@ def get_existing_query(self):
else:
raise ClickException(f"No query found with the id {query_id}")

def get_query_body(self):
def get_query_body(self, scheduled):
scheduled_body = self.create_scheduled_body(scheduled)
body_q = {
"kind": "doubleclickbidmanager#query",
"metadata": {
Expand All @@ -87,18 +91,29 @@ def get_query_body(self):
"metrics": list(self.kwargs.get("query_metric", [])),
"filters": [{"type": filt[0], "value": str(filt[1])} for filt in self.kwargs.get("filter")],
},
"schedule": {"frequency": self.kwargs.get("query_frequency", "ONE_TIME")},
"schedule": scheduled_body,
}
if self.kwargs.get("start_date") is not None and self.kwargs.get("end_date") is not None:
if not scheduled and self.kwargs.get("start_date") is not None and self.kwargs.get("end_date") is not None:
body_q["metadata"]["dataRange"] = "CUSTOM_DATES"
body_q["reportDataStartTimeMs"] = 1000 * int(
(self.kwargs.get("start_date") + datetime.timedelta(days=1)).timestamp()
)
body_q["reportDataEndTimeMs"] = 1000 * int((self.kwargs.get("end_date") + datetime.timedelta(days=1)).timestamp())
return body_q

def create_and_get_query(self):
body_query = self.get_query_body()
def create_scheduled_body(self, scheduled):
if not scheduled:
return {"frequency": "ONE_TIME"}
else:
return {
"frequency": self.kwargs.get("query_frequency"),
"nextRunTimezoneCode": self.kwargs.get("query_timezone_code"),
"endTimeMs": 1000 * int((self.kwargs.get("end_date") + datetime.timedelta(days=1)).timestamp()),
"startTimeMs": 1000 * int((self.kwargs.get("start_date") + datetime.timedelta(days=1)).timestamp()),
}

def create_and_get_query(self, scheduled=False):
body_query = self.get_query_body(scheduled)
query = self._client.queries().createquery(body=body_query).execute()
return query

Expand Down Expand Up @@ -176,7 +191,9 @@ def read(self):
if request_type == "existing_query":
data = [self.get_existing_query()]
elif request_type == "custom_query":
data = [self.create_and_get_query()]
data = [self.create_and_get_query(scheduled=False)]
elif request_type == "custom_scheduled_query":
data = [self.create_and_get_query(scheduled=True)]
elif request_type == "existing_query_report":
data = self.get_query_report(existing_query=True)
elif request_type == "custom_query_report":
Expand Down
37 changes: 34 additions & 3 deletions tests/readers/google_dbm/test_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def mock_dbm_reader(self, **kwargs):
setattr(self, param, value)

@mock.patch.object(GoogleDBMReader, "__init__", mock_dbm_reader)
def test_get_query_body(self):
def test_get_query_body_not_scheduled(self):
kwargs = {}
reader = GoogleDBMReader(**kwargs)
reader.kwargs = {"filter": [("FILTER_ADVERTISER", 1)]}
Expand All @@ -46,7 +46,7 @@ def test_get_query_body(self):
"schedule": {"frequency": "ONE_TIME"},
}

self.assertDictEqual(reader.get_query_body(), expected_query_body)
self.assertDictEqual(reader.get_query_body(scheduled=False), expected_query_body)

@mock.patch.object(GoogleDBMReader, "__init__", mock_dbm_reader)
def test_get_query_body_ms_conversion(self):
Expand All @@ -71,4 +71,35 @@ def test_get_query_body_ms_conversion(self):
"reportDataStartTimeMs": 1579132800000,
"reportDataEndTimeMs": 1579392000000,
}
self.assertDictEqual(reader.get_query_body(), expected_query_body)
self.assertDictEqual(reader.get_query_body(scheduled=False), expected_query_body)

@mock.patch.object(GoogleDBMReader, "__init__", mock_dbm_reader)
def test_get_scheduled_query_body_ms_conversion(self):
kwargs = {}
reader = GoogleDBMReader(**kwargs)
reader.kwargs = {
"filter": [("FILTER_ADVERTISER", 1)],
"start_date": datetime.datetime(2020, 1, 15, tzinfo=datetime.timezone.utc),
"end_date": datetime.datetime(2020, 1, 18, tzinfo=datetime.timezone.utc),
"day_range": "LAST_7_DAYS",
"query_timezone_code": "America/New_York",
"query_frequency": "DAILY",
}

expected_query_body = {
"kind": "doubleclickbidmanager#query",
"metadata": {"format": "CSV", "title": "NO_TITLE_GIVEN", "dataRange": "LAST_7_DAYS"},
"params": {
"type": "TYPE_TRUEVIEW",
"groupBys": [],
"metrics": [],
"filters": [{"type": "FILTER_ADVERTISER", "value": "1"}],
},
"schedule": {
"frequency": "DAILY",
"nextRunTimezoneCode": "America/New_York",
"endTimeMs": 1579392000000,
"startTimeMs": 1579132800000,
},
}
self.assertDictEqual(reader.get_query_body(scheduled=True), expected_query_body)