Merge pull request #56 from artefactory/dev

NCK v1.2

.dockerignore

@@ -0,0 +1,14 @@
+**/.github
+**/tests
+**/.env
+**/.flake8
+**/.gitignore
+**/.git
+CONTRIBUTING.md
+README.md
+**/__pycache__
+**/*.pyc
+**/.settings
+**/.vscode
+**/Dockerfile*
+**/requirements-dev.txt

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+  - repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+      - id: black
+        entry: black
+        language_version: python3.7
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.2.3
+    hooks:
+      - id: flake8

README.md

@@ -22,6 +22,7 @@ Nautilus connectors kit is a tool which aim is getting raw data from different s
 - MySQL
 - Radarly
 - SalesForce
+- The Trade Desk
 - Twitter Ads
 - Yandex Campaign
 - Yandex Statistics

nck/entrypoint.py

@@ -59,7 +59,6 @@ def run(processors, state_service_name, state_service_host, state_service_port,
         raise click.BadParameter("You must specify at least one writer")
 
     reader = _readers[0]
-
     # A stream should represent a full file!
     for stream in reader.read():
         for writer in _writers:

nck/helpers/dbm_helper.py

@@ -21,23 +21,5 @@
     "existing_query_report",
     "custom_query_report",
     "lineitems_objects",
-    "sdf_objects",
     "list_reports",
 ]
-
-POSSIBLE_SDF_FILE_TYPES = [
-    "INVENTORY_SOURCE",
-    "AD",
-    "AD_GROUP",
-    "CAMPAIGN",
-    "INSERTION_ORDER",
-    "LINE_ITEM",
-]
-
-FILE_TYPES_DICT = {
-    "AD": "ads",
-    "AD_GROUP": "adGroups",
-    "CAMPAIGN": "campaigns",
-    "LINE_ITEM": "lineItems",
-    "INSERTION_ORDER": "insertionOrders",
-}

nck/helpers/dv360_helper.py

@@ -0,0 +1,20 @@
+FILE_NAMES = {
+    "FILE_TYPE_INSERTION_ORDER": "InsertionOrders",
+    "FILE_TYPE_CAMPAIGN": "Campaigns",
+    "FILE_TYPE_MEDIA_PRODUCT": "MediaProducts",
+    "FILE_TYPE_LINE_ITEM": "LineItems",
+    "FILE_TYPE_AD_GROUP": "AdGroups",
+    "FILE_TYPE_AD": "AdGroupAds"
+}
+
+FILE_TYPES = FILE_NAMES.keys()
+
+FILTER_TYPES = [
+    "FILTER_TYPE_UNSPECIFIED",
+    "FILTER_TYPE_NONE",
+    "FILTER_TYPE_ADVERTISER_ID",
+    "FILTER_TYPE_CAMPAIGN_ID",
+    "FILTER_TYPE_MEDIA_PRODUCT_ID",
+    "FILTER_TYPE_INSERTION_ORDER_ID",
+    "FILTER_TYPE_LINE_ITEM_ID"
+]

nck/helpers/ttd_helper.py

@@ -0,0 +1,64 @@
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+import logging
+from datetime import datetime
+
+API_HOST = "https://api.thetradedesk.com/v3"
+
+API_ENDPOINTS = {
+    "get_report_template_id": ("POST", "myreports/reporttemplateheader/query"),
+    "create_report_schedule": ("POST", "myreports/reportschedule"),
+    "get_report_execution_details": (
+        "POST",
+        "myreports/reportexecution/query/advertisers",
+    ),
+    "delete_report_schedule": ("DELETE", "/myreports/reportschedule"),
+}
+
+DEFAULT_REPORT_SCHEDULE_ARGS = {
+    "ReportFileFormat": "CSV",
+    "ReportDateRange": "Custom",
+    "TimeZone": "UTC",
+    "ReportDateFormat": "Sortable",
+    "ReportNumericFormat": "US",
+    "IncludeHeaders": True,
+    "ReportFrequency": "Once",
+}
+
+DEFAULT_PAGING_ARGS = {
+    "PageStartIndex": 0,
+    "PageSize": 10,
+}
+
+API_DATEFORMAT = "%Y-%m-%dT%H:%M:%S"
+BQ_DATEFORMAT = "%Y-%m-%d"
+
+
+class ReportTemplateNotFoundError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+        logging.error(message)
+
+
+class ReportScheduleNotReadyError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+        logging.error(message)
+
+
+def format_date(date_string):
+    """
+    Input: "2020-01-01T00:00:00"
+    Output: "2020-01-01"
+    """
+    return datetime.strptime(date_string, API_DATEFORMAT).strftime(BQ_DATEFORMAT)

nck/helpers/twitter_helper.py

@@ -0,0 +1,72 @@
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+from twitter_ads.campaign import FundingInstrument, Campaign, LineItem
+from twitter_ads.creative import MediaCreative, PromotedTweet, CardsFetch
+
+
+REPORT_TYPES = ["ANALYTICS", "REACH", "ENTITY"]
+
+ENTITY_OBJECTS = {
+    "FUNDING_INSTRUMENT": FundingInstrument,
+    "CAMPAIGN": Campaign,
+    "LINE_ITEM": LineItem,
+    "MEDIA_CREATIVE": MediaCreative,
+    "PROMOTED_TWEET": PromotedTweet,
+}
+
+ENTITY_ATTRIBUTES = {
+    **{
+        entity: list(ENTITY_OBJECTS[entity].__dict__["PROPERTIES"].keys())
+        for entity in ENTITY_OBJECTS
+    },
+    "CARD": list(CardsFetch.__dict__["PROPERTIES"].keys()),
+}
+
+GRANULARITIES = ["DAY", "TOTAL"]
+
+METRIC_GROUPS = [
+    "ENGAGEMENT",
+    "BILLING",
+    "VIDEO",
+    "MEDIA",
+    "MOBILE_CONVERSION",
+    "WEB_CONVERSION",
+    "LIFE_TIME_VALUE_MOBILE_CONVERSION",
+]
+
+PLACEMENTS = [
+    "ALL_ON_TWITTER",
+    "PUBLISHER_NETWORK",
+]
+
+SEGMENTATION_TYPES = [
+    "AGE",
+    "APP_STORE_CATEGORY",
+    "AUDIENCES",
+    "CONVERSATIONS",
+    "CONVERSION_TAGS",
+    "DEVICES",
+    "EVENTS",
+    "GENDER",
+    "INTERESTS",
+    "KEYWORDS",
+    "LANGUAGES",
+    "LOCATIONS",
+    "METROS",
+    "PLATFORMS",
+    "PLATFORM_VERSIONS",
+    "POSTAL_CODES",
+    "REGIONS",
+    "SIMILAR_TO_FOLLOWERS_OF_USER",
+    "TV_SHOWS",
+]