Merge pull request #32 from artefactory/dev

NCK v1.1.1

nck/helpers/adobe_helper.py

@@ -16,9 +16,6 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 import datetime
-import binascii
-import uuid
-import hashlib
 import more_itertools
 import logging
 from nck.utils.text import reformat_naming_for_bq
@@ -28,31 +25,10 @@
 # github : https://github.com/SaturnFromTitan/adobe_analytics
 
 
-def _serialize_header(properties):
-    header = ['{key}="{value}"'.format(key=k, value=v) for k, v in properties.items()]
-    return ", ".join(header)
-
-
-def build_headers(password, username):
-    nonce = str(uuid.uuid4())
-    base64nonce = binascii.b2a_base64(binascii.a2b_qp(nonce))
-    created_date = datetime.datetime.utcnow().isoformat() + "Z"
-    sha = nonce + created_date + password
-    sha_object = hashlib.sha1(sha.encode())
-    password_64 = binascii.b2a_base64(sha_object.digest())
-
-    properties = {
-        "Username": username,
-        "PasswordDigest": password_64.decode().strip(),
-        "Nonce": base64nonce.decode().strip(),
-        "Created": created_date,
-    }
-    header = "UsernameToken " + _serialize_header(properties)
-    return {"X-WSSE": header}
-
-
 def _parse_header(report):
-    dimensions = [_classification_or_name(dimension) for dimension in report["elements"]]
+    dimensions = [
+        _classification_or_name(dimension) for dimension in report["elements"]
+    ]
     metrics = [metric["name"] for metric in report["metrics"]]
     return dimensions, metrics
 
@@ -125,12 +101,19 @@ def _dimension_value(chunk):
 
 
 def _dimension_value_is_nan(chunk):
-    return ("name" not in chunk) or (chunk["name"] == "") or (chunk["name"] == "::unspecified::")
+    return (
+        ("name" not in chunk)
+        or (chunk["name"] == "")
+        or (chunk["name"] == "::unspecified::")
+    )
 
 
 def _to_datetime(chunk):
     time_stamp = datetime.datetime(
-        year=chunk["year"], month=chunk["month"], day=chunk["day"], hour=chunk.get("hour", 0)
+        year=chunk["year"],
+        month=chunk["month"],
+        day=chunk["day"],
+        hour=chunk.get("hour", 0),
     )
     return time_stamp.strftime("%Y-%m-%d %H:00:00")
 
@@ -150,6 +133,12 @@ def parse(raw_response):
                 yield {header: None for header in headers}
 
 
+class ReportDescriptionError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+        logging.error(message)
+
+
 class ReportNotReadyError(Exception):
     def __init__(self, message):
         super().__init__(message)

nck/helpers/adobe_helper_2_0.py

@@ -19,9 +19,6 @@
 import logging
 from datetime import datetime
 
-logging.basicConfig(level="INFO")
-logger = logging.getLogger()
-
 
 class APIRateLimitError(Exception):
     def __init__(self, message):

nck/helpers/facebook_helper.py

@@ -16,6 +16,10 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
+import logging
+import json
+from time import sleep
+
 from facebook_business.adobjects.adsinsights import AdsInsights
 
 FACEBOOK_OBJECTS = ["creative", "ad", "adset", "campaign", "account"]
@@ -155,3 +159,43 @@ def get_field_values(resp_obj, field_path, action_breakdowns, visited=[]):
             return get_all_action_breakdown_values(
                 resp_obj, visited[:-1], action_breakdowns, filters
             )
+
+
+def generate_batches(iterable, batch_size):
+    """
+    Yields lists of length size batch_size,
+    containing objects yielded by the iterable.
+    """
+
+    batch = []
+    for obj in iterable:
+        if len(batch) == batch_size:
+            yield batch
+            batch = []
+        batch.append(obj)
+
+    if len(batch):
+        yield batch
+
+
+def monitor_usage(response):
+    """
+    Extracts "X-Business-Use-Case-Usage" header from a FacebookResponse object.
+    If one of the 3 API usage rates (call_count, total_cputime, total_time)
+    is above 75%, puts the program to sleep for 5 minutes.
+    Documentation: https://developers.facebook.com/docs/graph-api/overview/rate-limiting/
+    """
+
+    for header in response._headers:
+        if header["name"] == "X-Business-Use-Case-Usage":
+            usage_header = json.loads(header["value"])
+            usage_header_values = list(usage_header.values())[0][0]
+            usage_rates = [
+                v
+                for k, v in usage_header_values.items()
+                if k in ["call_count", "total_cputime", "total_time"]
+            ]
+
+    if max(usage_rates) > 75:
+        logging.info("75% rate limit reached. Sleeping for 5 minutes...")
+        sleep(300)